
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns


PEA15 = pd.read_csv('Data/CSO_PEA15_Population_Change.csv', sep = ',')


PEA15.head(n=5)
PEA15.tail(n=5)


#PEA15 VALUE times 1000
PEA15['VALUE'] = PEA15['VALUE']*1000


#PEA15 remove STATISTIC Label column and UNIT column
PEA15 = PEA15.drop(columns = ['STATISTIC Label', 'UNIT'])


#PEA15 transpose Component column
PEA15 = PEA15.pivot_table(index = 'Year', columns = 'Component', values = 'VALUE').reset_index()


# Name of first column to 'Index'
PEA15.columns.name = 'Index'


PEA15.head(n=8)


# Graph annual births, annual deaths, immigrants and emigrants per year
PEA15.plot(x = 'Year', y = ['Annual births', 'Annual deaths', 'Emigrants', 'Immigrants', 'Net migration'], xlabel='Year', title='Population Changes in Ireland from 1987 to 2023')

<Axes: title={'center': 'Population Changes in Ireland from 1987 to 2023'}, xlabel='Year'>


PEA15.plot(x = 'Year', y = [ 'Emigrants', 'Immigrants', 'Net migration'], xlabel='Year', title='Population Changes in Ireland from 1987 to 2023')

<Axes: title={'center': 'Population Changes in Ireland from 1987 to 2023'}, xlabel='Year'>


PEA15.plot(x = 'Year', y = [ 'Emigrants', 'Immigrants'], xlabel='Year', title='Population Changes in Ireland from 1987 to 2023')

<Axes: title={'center': 'Population Changes in Ireland from 1987 to 2023'}, xlabel='Year'>


PEA03 = pd.read_csv('Data/CSO_PEA03_Migration_Sex_Age.csv', sep = ',')


PEA03.head(n=5)


# values for STATISTIC Label column in PEA03
PEA03['STATISTIC Label'].unique()

array(['Estimated Migration (Persons in April)'], dtype=object)


#PEA03 VALUE times 1000
PEA03['VALUE'] = PEA03['VALUE']*1000


# Remove STATISTIC Label column from PEA03
PEA03 = PEA03.drop(columns = ['STATISTIC Label', "UNIT"])


#PEA03 transpose Component column
PEA03 = PEA03.pivot_table(index = ['Year', 'Age Group', 'Sex'], columns = 'Inward or Outward Flow', values = 'VALUE').reset_index()


# Name of first column to 'Index'
PEA03.columns.name = 'Index'


PEA03.head(n=25)


# Change name columns 3 and 4
PEA03 = PEA03.rename(columns = {'Emigrants: All destinations': 'Emigrants'})
PEA03 = PEA03.rename(columns = {'Immigrants: All origins': 'Immigrants'})


# Emigration and Immigration for all ages and sexes
PEA03[(PEA03['Age Group'] == 'All ages') & (PEA03['Sex'] == 'Both sexes')].plot(x='Year', y=['Emigrants', 'Immigrants'], xlabel='Year', title='Migration in Ireland from 1987 to 2023')

<Axes: title={'center': 'Migration in Ireland from 1987 to 2023'}, xlabel='Year'>


# Immigration by Sex
sns.lineplot(data=PEA03[(PEA03['Age Group'] == 'All ages') & (PEA03['Sex'].isin(['Female', 'Male']))], x='Year', y='Immigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Immigration to Ireland by Sex from 2002 to 2023')
plt.legend(title='Sex')
plt.show()


# Emigration by Sex
sns.lineplot(data=PEA03[(PEA03['Age Group'] == 'All ages') & (PEA03['Sex'].isin(['Female', 'Male']))], x='Year', y='Emigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Emigration from Ireland by Sex from 2002 to 2023')
plt.legend(title='Sex')
plt.show()


# Immigration by Age Group
sns.lineplot(data=PEA03[(PEA03['Age Group'] != 'All ages') & (PEA03['Sex'] == 'Both sexes')], x='Year', y='Immigrants', hue='Age Group')
plt.xlabel('Year')
plt.title('Immigration to Ireland by Age Group from 1987 to 2023')
plt.legend(title='Age Group', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()

# similar trend for immigration for 15-24 and 25-44 age groups untill 2010, but after divide with 25-44 age group having increasingly higher immigration and 
# 15-24 age group becoming similar in trends with other age groups


# Emigration by Age Group
sns.lineplot(data=PEA03[(PEA03['Age Group'] != 'All ages') & (PEA03['Sex'] == 'Both sexes')], x='Year', y='Emigrants', hue='Age Group')
plt.xlabel('Year')
plt.title('Emigration from Ireland by Age Group from 1987 to 2023')
plt.legend(title='Age Group', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()
# age group 25-44 has gotten higher emigration over the yaers aswell as the 0-14 group, which could mean that adults with young children are leaving the country
# especially from years 2010 to 2016
# older people tend to not leave ireland as much as younger people (consistent over years)


PEA18 = pd.read_csv('Data/CSO_PEA18_Migration_Sex_Country.csv', sep = ',')


PEA18.head(n=5)


# values for STATISTIC Label column in PEA18
PEA18['STATISTIC Label'].unique()

array(['Estimated Migration (Persons in April)'], dtype=object)


#PEA18 VALUE times 1000
PEA18['VALUE'] = PEA18['VALUE']*1000


# Remove STATISTIC Label column from PEA18
PEA18 = PEA18.drop(columns = ['STATISTIC Label', "UNIT"])


#PEA18 transpose Component column
PEA18 = PEA18.pivot_table(index = ['Year', 'Country', 'Sex'], columns = 'Origin or Destination', values = 'VALUE').reset_index()


# Change name 1st column
PEA18.columns.name = 'Index'


# Change name columns for destination or origin
PEA18 = PEA18.rename(columns = {'Emigrants: All destinations': 'Emigrants'})
PEA18 = PEA18.rename(columns = {'Immigrants: All origins': 'Immigrants'})


PEA18.head(n=15)


# values for country column in PEA18
PEA18['Country'].unique()

array(['All countries', 'EU14 excl Irl (UK & Ireland)',
       'Other countries (23)', 'United Kingdom (1)', 'United States',
       'EU15 to EU27 (accession countries joined post 2004)', 'Australia',
       'Canada'], dtype=object)


# Immigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] != 'All countries') & (PEA18['Sex'] == 'Both sexes')], x='Year', y='Immigrants', hue='Country')
plt.xlabel('Year')
plt.title('Immigration by Country from 1987 to 2023')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()

# Spike in immigration from other countries outside eu, uk, us, australia and canada after 2021 (after covid)
# immigration numbers for this were however already steadily increasing from 2010


# Immigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'All countries') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Immigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Immigration by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Immigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'All countries') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Emigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Emmigration by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Immigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'Other countries (23)') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Immigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Immigration from *Other countries* to Ireland by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Emigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'Other countries (23)') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Emigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Emigration into *Other countries* from Ireland by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Immigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'United States') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Immigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Immigration from US to Ireland by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Emigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'United States') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Emigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Emigration from Ireland to US by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Immigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'EU14 excl Irl (UK & Ireland)') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Immigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Immigration from EU14 to Ireland by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Emigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'EU14 excl Irl (UK & Ireland)') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Emigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Emigration from Ireland to EU14 by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Immigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'United Kingdom (1)') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Immigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Immigration from the UK to Ireland by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Emigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] == 'United Kingdom (1)') & (PEA18['Sex'] != 'Both sexes')], x='Year', y='Emigrants', hue='Sex')
plt.xlabel('Year')
plt.title('Emigration into the UK from Ireland by Sex from 1987 to 2023')
plt.legend(title='Country')
plt.show()
# No real divide by sex


# Emigration by Country
sns.lineplot(data=PEA18[(PEA18['Country'] != 'All countries') & (PEA18['Sex'] == 'Both sexes')], x='Year', y='Emigrants', hue='Country')
plt.xlabel('Year')
plt.title('Emigration by Country from 1987 to 2023')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


PEA18_2 = PEA18[(PEA18['Sex'] == 'Both sexes') & (PEA18['Country'].isin(['All countries', 'EU14 excl Irl (UK & Ireland)', 'Other countries (23)', 'United Kingdom (1)', 'United States', 'EU15 to EU27 (accession countries joined post 2004)']))]
PEA18_2.head()


PEA18_2.tail(n=10)


#PEA18_2 transpose Component column
PEA18_2 = PEA18_2.pivot_table(index = ['Year'], columns = ['Country'], values = ['Emigrants', 'Immigrants', 'Net migration']).reset_index()


PEA18_2.columns = ['_'.join(col).strip() for col in PEA18_2.columns.values]


# Change column names
PEA18_2 = PEA18_2.rename(columns = {'Year_':'Year'})


PEA18_2.tail(n=10)


PEA23 = pd.read_csv('Data/CSO_PEA23_Emigration_Sex_Citizenship.csv', sep = ',')


PEA23.head(n=5)


# values for STATISTIC Label column in PEA23
PEA23['Statistic Label'].unique()

array(['Estimated Emigration (Persons in April)'], dtype=object)


PEA23['Year'].unique()

array([2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016,
       2017, 2018, 2019, 2020, 2021, 2022, 2023], dtype=int64)


#PEA23 VALUE times 1000
PEA23['VALUE'] = PEA23['VALUE']*1000


# Remove STATISTIC Label column from PEA23
PEA23 = PEA23.drop(columns = ['Statistic Label', "UNIT"])


#PEA23 rename VALUE column to Emigrants
PEA23 = PEA23.rename(columns = {'VALUE': 'Emigrants'})


PEA23.head(n=25)


# filter citizzenship column on 'Ireland'
PEA23 = PEA23[PEA23['Citizenship'] == 'Ireland']


PEA23.head()


# import PEA 24
PEA24 = pd.read_csv('Data/CSO_PEA24_Immigration_Sex_Citizenship.csv', sep = ',')


PEA24.head(n=5)


# values for STATISTIC Label column in PEA24
PEA24['Statistic Label'].unique()

array(['Estimated Immigration (Persons in April)'], dtype=object)


#PEA24 VALUE times 1000
PEA24['VALUE'] = PEA24['VALUE']*1000


# Remove STATISTIC Label column from PEA24
PEA24 = PEA24.drop(columns = ['Statistic Label', "UNIT"])


#PEA24 rename VALUE column to Immigrants
PEA24 = PEA24.rename(columns = {'VALUE': 'Immigrants'})


PEA24.head(n=25)


# filter citizzenship column on 'Ireland'
PEA24 = PEA24[PEA24['Citizenship'] == 'Ireland']


PEA24.head()


# import CPM01
CPM01 = pd.read_csv('Data/CPM01_ConsumerPriceIndex.csv', sep = ',')


CPM01.head()


print(CPM01['UNIT'].unique())
print(CPM01['Statistic Label'].unique())

['Base Dec 2023=100' 'Base Dec 2016=100' 'Base Dec 2011=100'
 'Base Dec 2006=100' 'Base Dec 2001=100' 'Base Nov 1996=100' '%']
['Consumer Price Index (Base Dec 2023=100)'
 'Consumer Price Index (Base Dec 2016=100)'
 'Consumer Price Index (Base Dec 2011=100)'
 'Consumer Price Index (Base Dec 2006=100)'
 'Consumer Price Index (Base Dec 2001=100)'
 'Consumer Price Index (Base Nov 1996=100)'
 'Percentage Change over 1 month for Consumer Price Index'
 'Percentage Change over 12 months for Consumer Price Index']


# Split month column on year and month
CPM01[['Year', 'Month']] = CPM01['Month'].str.split(' ', expand=True)


#CPM01 transpose Component column
CPM01 = CPM01.pivot_table(index = ['Year', 'Commodity Group'], columns = 'Statistic Label', values = 'VALUE').reset_index()


# Take average per year
CPM01 = CPM01.groupby(['Year', 'Commodity Group']).mean().reset_index()


# Change name 1st column
CPM01.columns.name = 'Index'


CPM01['Year'] = pd.to_numeric(CPM01['Year'])


CPM01.head()


CPM01.columns.tolist()

['Year',
 'Commodity Group',
 'Consumer Price Index (Base Dec 2001=100)',
 'Consumer Price Index (Base Dec 2006=100)',
 'Consumer Price Index (Base Dec 2011=100)',
 'Consumer Price Index (Base Dec 2016=100)',
 'Consumer Price Index (Base Dec 2023=100)',
 'Consumer Price Index (Base Nov 1996=100)',
 'Percentage Change over 1 month for Consumer Price Index',
 'Percentage Change over 12 months for Consumer Price Index']


# Consumer Price Index 1975-2024
sns.lineplot(data=CPM01, x='Year', y='Consumer Price Index (Base Dec 2001=100)', hue='Commodity Group')
plt.xlabel('Year')
plt.title('Consumer Price Index')
plt.legend(title='Commodity Group', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


CPM01['Commodity Group'].unique()

array(['All items', 'Alcoholic beverages and tobacco',
       'Clothing and footwear', 'Communications', 'Education',
       'Food and non-alcoholic beverages',
       'Furnishings, household equipment and routine household maintenance',
       'Health', 'Housing, water, electricity, gas and other fuels',
       'Miscellaneous goods and services', 'Recreation and culture',
       'Restaurants and hotels', 'Transport'], dtype=object)


# Consumer Price Index 1975-2024
sns.lineplot(data=CPM01[(CPM01['Commodity Group'] == 'Food and non-alcoholic beverages') |
                        (CPM01['Commodity Group'] == 'Housing, water, electricity, gas and other fuels') |
                        (CPM01['Commodity Group'] == 'All items')],
             x='Year', y='Consumer Price Index (Base Dec 2001=100)', hue='Commodity Group')
plt.xlabel('Year')
plt.title('Consumer Price Index 1975-2024')
plt.legend(title='Commodity Group', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Consumer Price Index 1975-2024
sns.lineplot(data=CPM01[(CPM01['Commodity Group'] == 'Food and non-alcoholic beverages') |
                        (CPM01['Commodity Group'] == 'Housing, water, electricity, gas and other fuels') |
                        (CPM01['Commodity Group'] == 'All items')],
             x='Year', y='Percentage Change over 12 months for Consumer Price Index', hue='Commodity Group')
plt.xlabel('Year')
plt.title('Consumer Price Index 1975-2024')
plt.legend(title='Commodity Group', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# import TSA02
TSA02 = pd.read_csv('Data/CSO_TSA02_Import_Export.csv', sep = ',')


TSA02.head()


print(TSA02['UNIT'].unique())
print(TSA02['Statistic Label'].unique())

['Base 2010=100']
['Trade Price Index for Imports' 'Trade Price Index for Exports'
 'Terms of Trade Index' 'Trade Volume Index for Imports'
 'Trade Volume Index for Exports']


#TSA02 transpose Component column
TSA02 = TSA02.pivot_table(index = ['Year'], columns = 'Statistic Label', values = 'VALUE').reset_index()


# Change name 1st column
TSA02.columns.name = 'Index'


# change variable names
TSA02 = TSA02.rename(columns = {'Terms of Trade Index': 'Terms of Trade Index (Base 2010)', 
                                'Trade Price Index for Exports':'Trade Price Index Exports (Base 2010)', 
                                'Trade Price Index for Imports':'Trade Price Index Imports (Base 2010)', 
                                'Trade Volume Index for Exports': 'Trade Volume Index Exports (Base 2010)',
                                'Trade Volume Index for Imports': 'Trade Volume Index Imports (Base 2010)'})


TSA02.head()


TSA02.tail()


TSA02.plot(x='Year', 
           y=['Terms of Trade Index (Base 2010)', 'Trade Price Index Exports (Base 2010)', 'Trade Price Index Imports (Base 2010)', 'Trade Volume Index Exports (Base 2010)', 'Trade Volume Index Imports (Base 2010)'],
           xlabel='Year', 
           title='Trade Index 1930-2022')

<Axes: title={'center': 'Trade Index 1930-2022'}, xlabel='Year'>


TSA02[TSA02['Year']>1999].plot(x='Year', 
           y=['Trade Price Index Exports (Base 2010)', 'Trade Price Index Imports (Base 2010)'],
           xlabel='Year', 
           title='Trade Price Index 1930-2022')

<Axes: title={'center': 'Trade Price Index 1930-2022'}, xlabel='Year'>


TSA02[TSA02['Year']>1999].plot(x='Year', 
           y=['Trade Volume Index Exports (Base 2010)', 'Trade Volume Index Imports (Base 2010)'],
           xlabel='Year', 
           title='Trade Volume Index 1930-2022')

<Axes: title={'center': 'Trade Volume Index 1930-2022'}, xlabel='Year'>


# import BPA33
BPA33 = pd.read_csv('Data/CSO_BPA33_Direct_Investment.csv', sep = ',')


BPA33.head()


print(BPA33['UNIT'].unique())
print(BPA33['Year'].unique())
print(BPA33['Statistic Label'].unique())
print(BPA33['Type of Investment'].unique())
print(BPA33['Geographic Location'].unique())

['Euro Million']
[2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022]
['Direct Investment  Flows Abroad' 'Direct Investment  Flows in Ireland']
['All Direct Investment']
['All countries and international organisations' 'United States'
 'Asia (6)' 'EU28' 'Europe (3)' 'North America (1)']


# Investment Flows Abroad by Location
sns.lineplot(data=BPA33[(BPA33['Statistic Label'] == 'Direct Investment  Flows Abroad')],
              x='Year', 
              y='VALUE', 
              hue='Geographic Location')
plt.xlabel('Year')
plt.title('Investment Flows Abroad by Location')
plt.legend(title='Geographic Location', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Investment Flows in Ireland by Location
sns.lineplot(data=BPA33[(BPA33['Statistic Label'] == 'Direct Investment  Flows in Ireland')],
              x='Year', 
              y='VALUE', 
              hue='Geographic Location')
plt.xlabel('Year')
plt.title('Investment Flows in Ireland by Location')
plt.legend(title='Geographic Location', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


BPA33_2 = BPA33[BPA33['Geographic Location'].isin(['EU28', 'All countries and international organisations', 'Asia (6)', 'United States'])]
BPA33_2.head()


#BPA33 transpose Component column
BPA33_2 = BPA33_2.pivot_table(index = ['Year'], columns = ['Statistic Label', 'Geographic Location'], values = 'VALUE').reset_index()


BPA33_2.head()


BPA33_2.columns = ['_'.join(col).strip() for col in BPA33_2.columns.values]


# Change column names
BPA33_2 = BPA33_2.rename(columns = {'Year_':'Year'})


BPA33_2.head()


# import GFA01
GFA01 = pd.read_csv('Data/CSO_GFA01_General_Government_Transactions.csv', sep = ',')


GFA01.head()


print(GFA01['UNIT'].unique())
print(GFA01['Statistic Label'].unique())

['Euro Million']
['General Government Transactions ESA 2010: Revenue, Expenditure, Financing and Deficit']


print(GFA01['Item'].unique())

['General Government transactions - Revenue - ESA2010 Code (TR)'
 'General Government transactions - Revenue - Taxes and social contributions - ESA2010 Code ( D2 + D5 + D91 + D61)'
 'General Government transactions - Revenue - Taxes - ESA2010 Code (D2 + D5 + D91)'
 'General Government transactions - Revenue - Social contributions - ESA2010 Code (D61)'
 'General Government transactions - Revenue - Sales of goods and services - ESA2010 Code (P10)'
 'General Government transactions - Revenue - Investment income - ESA2010 Code (D4)'
 'General Government transactions - Revenue - Current transfer revenue (excluding taxes) - ESA2010 Code (D7)'
 'General Government transactions - Revenue - Capital transfer revenue (excluding taxes) - ESA2010 Code (D9N)'
 'General Government transactions - Expenditure - ESA2010 Code (TE)'
 'General Government transactions - Expenditure - Expense - ESA2010 Code (Expense)'
 'General Government transactions - Expenditure - Compensation of employees - ESA2010 Code (D1)'
 'General Government transactions - Expenditure - Use of goods and services plus taxes payable - ESA2010 Code (P2 + D5 + D29)'
 'General Government transactions - Expenditure - Depreciation (consumption of fixed capital) - ESA2010 Code (P51c)'
 'General Government transactions - Expenditure - Interest (excluding FISIM) - ESA2010 Code (D41)'
 'General Government transactions - Expenditure - Subsidies - ESA2010 Code (D3)'
 'General Government transactions - Expenditure - Social benefits - ESA2010 Code (D62 + D632)'
 'General Government transactions - Expenditure - Other current transfers - ESA2010 Code (D7)'
 'General Government transactions - Expenditure - Capital transfers - ESA2010 Code (D9)'
 'Net operating balance - ESA2010 Code  (NOB)'
 'Net acquisition of non-financial assets - ESA2010 Code (P5 + NP - P51c)'
 'General Government Surplus/Deficit - ESA2010 Code (B9)'
 'Net acquisition of financial assets'
 'Net acquisition of financial assets - Debt instruments - ESA2010 Code (F2 + F3 + F4)'
 'Net acquisition of financial assets - Equity and Investment Fund Shares - ESA2010 Code (F5)'
 'Net acquisition of financial assets - Other financial assets - ESA2010 Code (F6 + F7 + F8)'
 'Net incurrence of liabilities'
 'Net incurrence of liabilities - Debt instruments - ESA2010 Code (F2 + F3 + F4)'
 'Net incurrence of liabilities - Other liabilities - ESA2010 Code (F5 + F6 + F7 + F8)'
 'Memo: Net derivative flows on debt interest swap contracts']


# Filter Items
GFA01 = GFA01[GFA01['Item'].isin(['General Government transactions - Revenue - ESA2010 Code (TR)',
                                  'General Government transactions - Expenditure - ESA2010 Code (TE)',
                                  'Net operating balance - ESA2010 Code  (NOB)',
                                  'General Government Surplus/Deficit - ESA2010 Code (B9)',
                                  'Net acquisition of financial assets',
                                  'Net incurrence of liabilities'])]


GFA01.head()


#GFA01 transpose Component column
GFA01 = GFA01.pivot_table(index = ['Year'], columns = 'Item', values = 'VALUE').reset_index()


# rename column headers
GFA01 = GFA01.rename(columns = {'General Government Surplus/Deficit - ESA2010 Code (B9)': 'General Surplus/Deficit (Million Euros)',
                                'General Government transactions - Expenditure - ESA2010 Code (TE)': 'General Expenditure (Million Euros)',
                                'General Government transactions - Revenue - ESA2010 Code (TR)': 'General Revenue (Million Euros)',
                                'Net acquisition of financial assets': 'Net Acquisition Financial Assets (Million Euros)',
                                'Net incurrence of liabilities': 'Net Incurrence Liabilities (Million Euros)',
                                'Net operating balance - ESA2010 Code  (NOB)': 'Net Operating Balance (Million Euros)'})


# Change name 1st column
GFA01.columns.name = 'Index'


GFA01.head()


ax = GFA01.plot(x='Year', 
                y=['General Surplus/Deficit (Million Euros)', 
                   'General Expenditure (Million Euros)',
                   'General Revenue (Million Euros)',
                   'Net Acquisition Financial Assets (Million Euros)',
                   'Net Incurrence Liabilities (Million Euros)',
                   'Net Operating Balance (Million Euros)'
                  ],
                xlabel='Year', 
                title='Government Transactions 1995-2023')

ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))

plt.show()


# Filter Items
GFA01 = GFA01[['Year', 'General Surplus/Deficit (Million Euros)', 'General Expenditure (Million Euros)','Net Acquisition Financial Assets (Million Euros)']]


GFA01.tail()


# import BRA08
BRA08 = pd.read_csv('Data/CSO_BRA08_Number_Enterprises.csv', sep = ',')


BRA08.tail()


print(BRA08['UNIT'].unique())
print(BRA08['Statistic Label'].unique())

['Number']
['Active Enterprises']


# Number of Active Enterprises from 2008-2020
sns.lineplot(data=BRA08[(BRA08['Employment Size'] == 'All persons engaged size classes')],
              x='Year', 
              y='VALUE', 
              hue='County')
plt.xlabel('Year')
plt.title('Number of Active Enterprises from 2008-2020')
plt.legend(title='County', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Number of Active Enterprises with over 250 employees from 2008-2020
sns.lineplot(data=BRA08[(BRA08['Employment Size'] == '250 and over')],
              x='Year', 
              y='VALUE', 
              hue='County')
plt.xlabel('Year')
plt.title('Number of Active Enterprises with over 250 employees from 2008-2020')
plt.legend(title='County', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Filter Counties
BRA08 = BRA08[BRA08['County'].isin(['All Counties', 'Dublin'])]


BRA08.head()


#BRA08 transpose Component column
BRA08 = BRA08.pivot_table(index = ['Year'], columns = ['Employment Size', 'County'], values = 'VALUE').reset_index()


BRA08.columns = ['_'.join(col).strip() for col in BRA08.columns.values]


# Change column names
BRA08 = BRA08.rename(columns = {'Year_':'Year',
                                '250 and over_All Counties': 'Enterprises_Ireland_250+',
                                '250 and over_Dublin': 'Enterprises_Dublin_250+',
                                'All persons engaged size classes_All Counties': 'Enterprises_Ireland',
                                'All persons engaged size classes_Dublin': 'Enterprises_Dublin'})


BRA08.head()


# import OECD 1
OECD1 = pd.read_csv('Data/OECD_1_Sex_Employment.csv', sep = ',')


OECD1.head(n=5)


# select columns from OECD 1
OECD1 = OECD1[['Year', 'Country', 'Place of birth', 'Gender', 'Rate', 'Value']]


#OECD1 rename column
OECD1 = OECD1.rename(columns = {'Place of birth': 'Place_of_Birth'})


#OECD1 transpose Rate column
OECD1 = OECD1.pivot_table(index = ['Year', 'Country', 'Place_of_Birth', 'Gender'], columns = 'Rate', values = 'Value').reset_index()


# Name of first column to 'Index'
OECD1.columns.name = 'Index'


OECD1.head(n=5)


OECD1['Country'].unique()

array(['Australia', 'Austria', 'Belgium', 'Czechia', 'Denmark', 'Estonia',
       'Finland', 'France', 'Germany', 'Greece', 'Hungary', 'Iceland',
       'Ireland', 'Italy', 'Luxembourg', 'Netherlands', 'Norway',
       'Poland', 'Portugal', 'Slovak Republic', 'Slovenia', 'Spain',
       'Sweden', 'Switzerland', 'United Kingdom', 'United States',
       'New Zealand', 'OECD - Total', 'Mexico', 'Chile', 'Canada',
       'Türkiye', 'Israel', 'Latvia', 'Lithuania'], dtype=object)


European_countries = ['Austria', 'Belgium', 'Czechia', 'Denmark', 'Estonia',
       'Finland', 'France', 'Germany', 'Greece', 'Hungary', 'Iceland',
       'Italy', 'Luxembourg', 'Netherlands', 'Norway',
       'Poland', 'Portugal', 'Slovak Republic', 'Slovenia', 'Spain',
       'Sweden', 'Switzerland', 'United Kingdom', 'Latvia', 'Lithuania']

WestEuropean_countries = ['Austria', 'Belgium', 'Denmark', 'France', 'Germany',
       'Luxembourg', 'Netherlands', 'Switzerland', 'United Kingdom']


# european average of unemployment from European_countries
OECD1_Europe = OECD1[OECD1['Country'].isin(European_countries)]
OECD1_WestEurope = OECD1[OECD1['Country'].isin(WestEuropean_countries)]


# Rates Native Irish
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate')
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Participation rate', label='Participation rate')
sns.lineplot(data=OECD1[(OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Unemployment rate', label='Unemployment rate')
plt.axhline(y=70, color='black', linestyle='--')
plt.axhline(y=5, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Rates for Irish in Ireland from 2000 to 2022')
plt.legend(title='Rates')
plt.show()


# Rates Native Irish
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate Ireland Native')
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Foreign-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate Ireland Foreign')
sns.lineplot(data=OECD1_Europe[(OECD1_Europe['Place_of_Birth'] == 'Foreign-born') & (OECD1_Europe['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate Europe Foreign')
sns.lineplot(data=OECD1[(OECD1['Country'] == 'United States')  & (OECD1['Place_of_Birth'] == 'Foreign-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate US Foreign')
plt.xlabel('Year')
plt.title('Employment rates from 2000 to 2022')
plt.legend(title='Rates', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Employment Rate Native Irish by Gender
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] != 'Total')], 
             x='Year', y='Employment rate', hue='Gender')
plt.axhline(y=70, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Employment Rate for Irish in Ireland from 2000 to 2022')
plt.legend(title='Gender')
plt.show()


# Rates Foreign Irish
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Foreign-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate')
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Foreign-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Participation rate', label='Participation rate')
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Foreign-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Unemployment rate', label='Unemployment rate')
plt.axhline(y=70, color='black', linestyle='--')
plt.axhline(y=5, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Rates for Non-Irish in Ireland from 2000 to 2022')
plt.legend(title='Rate')
plt.show()


# Employment Rate Foreign Irish by Gender
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Ireland') & (OECD1['Place_of_Birth'] == 'Foreign-born') & (OECD1['Gender'] != 'Total')], 
             x='Year', y='Employment rate', hue='Gender')
plt.axhline(y=70, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Employment Rate for Non-Irish in Ireland from 2000 to 2022')
plt.legend(title='Gender')
plt.show()


# Rates Native Foreigners Average
sns.lineplot(data=OECD1[(OECD1['Country'] != 'OECD - Total') & (OECD1['Country'] != 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate')
sns.lineplot(data=OECD1[(OECD1['Country'] != 'OECD - Total') & (OECD1['Country'] != 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Participation rate', label='Participation rate')
sns.lineplot(data=OECD1[(OECD1['Country'] != 'OECD - Total') & (OECD1['Country'] != 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Unemployment rate', label='Unemployment rate')
plt.axhline(y=70, color='black', linestyle='--')
plt.axhline(y=5, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Rates for Natives outside Ireland from 2000 to 2022')
plt.legend(title='Rate')
plt.show()


# Employment Rate Native Foreigners by Gender
sns.lineplot(data=OECD1[(OECD1['Country'] != 'OECD - Total') & (OECD1['Country'] != 'Ireland') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] != 'Total')], 
             x='Year', y='Employment rate', hue='Gender')
plt.axhline(y=70, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Employment Rate for Natives outside Ireland from 2000 to 2022')
plt.legend(title='Gender')
plt.show()


# Rates Native Canadians Average
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Canada') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Employment rate', label='Employment rate')
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Canada') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Participation rate', label='Participation rate')
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Canada') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] == 'Total')], 
             x='Year', y='Unemployment rate', label='Unemployment rate')
plt.axhline(y=70, color='black', linestyle='--')
plt.axhline(y=5, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Rates for Natives Canada from 2008 to 2022')
plt.legend(title='Rate')
plt.show()


# Employment Rate Native Canadians by Gender
sns.lineplot(data=OECD1[(OECD1['Country'] == 'Canada') & (OECD1['Place_of_Birth'] == 'Native-born') & (OECD1['Gender'] != 'Total')], 
             x='Year', y='Employment rate', hue='Gender')
plt.axhline(y=70, color='black', linestyle='--')
plt.xlabel('Year')
plt.title('Rates for Natives Canada from 2008 to 2022')
plt.legend(title='Gender')
plt.show()


# import OECD 2
OECD2 = pd.read_csv('Data/OECD_2_Immigrants_Sector.csv', sep = ',')


OECD2.head(n=5)


# Check unique values Country of Residence
OECD2['Country of residence'].unique()
# only people living in Ireland

array(['Ireland'], dtype=object)


# select columns from OECD 2
OECD2 = OECD2[['Country of birth', 'Education level', 'Sector', 'Sex', 'Value']]


#OECD2 rename columns
OECD2 = OECD2.rename(columns = {'Country of birth': 'Continent_of_Birth','Education level': 'Education_Level', 'Sex': 'Gender'})


OECD2.head(n=5)


# Check unique values Country of Birth
OECD2['Continent_of_Birth'].unique()
# Continents and All countries

# OECD2['Country_of_Birth'] != 'All countries of birth' & OECD2['Country_of_Birth'] != 'Other and unknown places of birth'

array(['Africa', 'Asia', 'Europe', 'North America', 'Oceania',
       'South and Central America and Caribbean',
       'Other and unknown places of birth', 'All countries of birth'],
      dtype=object)


# change South and Central America and Caribbean to S&C America and Caribbean
OECD2.loc[OECD2['Continent_of_Birth'] == 'South and Central America and Caribbean', 'Continent_of_Birth'] = 'S&C America & Caribbean'


OECD2_Continents = OECD2[(OECD2['Continent_of_Birth'] != 'All countries of birth') & (OECD2['Continent_of_Birth'] != 'Other and unknown places of birth')]


# Check unique values Education Level & Sector
print(OECD2['Education_Level'].unique())
# no education and three levels of education

print(OECD2['Sector'].unique())
print(len(OECD2['Sector'].unique()))
# 18 sectors
# find out which sectors important to look at --> which sectors have most migration around current years

# OECD2['Sector'] == 'Financial intermediation' & OECD2['Sector'] == 'Real estate, renting and business activities' & OECD2['Sector'] == 'Transport, storage and communications'

['ISCED 3/4' 'Unknown education' 'ISCED 0/1/2' 'ISCED 5/6'
 'All levels of education']
['Electricity, gas and water supply'
 'Wholesale and retail trade; repair of motor vehicles, motorcycles and personal and household goods'
 'Hotels and restaurants'
 'Public administration and defence; compulsory social security'
 'Extra-territorial organizations and bodies' 'Construction'
 'Financial intermediation' 'Mining and quarrying' 'Education'
 'Manufacturing' 'Other community, social and personal service activities'
 'Private households with employed persons'
 'Transport, storage and communications' 'Agriculture and fishing'
 'Real estate, renting and business activities' 'Health and social work'
 'Unknown' 'All sectors']
18


# average value for each sector
sns.barplot(data=OECD2_Continents[(OECD2_Continents['Sector'] != 'All sectors') & (OECD2_Continents['Education_Level'] == 'All levels of education') & (OECD2_Continents['Gender'] == 'Men and women')], 
             y='Sector', x='Value', estimator=sum)

<Axes: xlabel='Value', ylabel='Sector'>


# Table overview per sector and continent
# Calculate sum of 'Value' column for each sector within each continent
table_data = OECD2_Continents[(OECD2_Continents['Sector'] != 'All sectors') & 
                              (OECD2_Continents['Education_Level'] == 'All levels of education') & 
                              (OECD2_Continents['Gender'] == 'Men and women')].pivot_table(
                                  index='Sector', columns='Continent_of_Birth', values='Value', aggfunc='sum')

# Generate a different color palette for each continent
palette = sns.color_palette("vlag", len(table_data.columns))

# Create a heatmap with numbers displayed in each cell and separate color scales per continent
sns.heatmap(table_data, cmap=palette, annot=True, fmt='.0f')

plt.show()


# numbers per Continent per Sector

# Calculate sum of 'Value' column for each sector within each continent
table_data = OECD2_Continents[(OECD2_Continents['Sector'] != 'All sectors') & 
                              (OECD2_Continents['Education_Level'] == 'All levels of education') & 
                              (OECD2_Continents['Gender'] == 'Men and women')].pivot_table(
                                  index='Sector', columns='Continent_of_Birth', values='Value', aggfunc='sum')

# Create an empty list to store color palettes
palettes = []

# Iterate over the columns (continents) and generate a unique color palette for each
for column in table_data.columns:
    palette = sns.color_palette("vlag", n_colors=len(table_data[column]))
    palettes.append(palette)

# Set figure size
plt.figure(figsize=(15, 5))

# Create a separate heatmap for each column with a different color palette
for i, (column, palette) in enumerate(zip(table_data.columns, palettes), 1):
    ax = plt.subplot(1, len(table_data.columns), i)
    if i == 1:
        sns.heatmap(table_data[[column]], cmap=palette, annot=True, fmt='.0f', cbar=False, yticklabels=True, ax=ax)
    else:
        sns.heatmap(table_data[[column]], cmap=palette, annot=True, fmt='.0f', cbar=False, yticklabels=False, ax=ax)
    plt.title(column)
    plt.yticks(rotation=0)  # Rotate y-axis tick labels for readability
    ax.set_xlabel('')  # Remove x label
    ax.set_ylabel('')  # Remove y label

plt.tight_layout()  # Adjust spacing between subplots
plt.suptitle("Number of Immigrants per Sector and Continent around Year 2000", y=1.05)
plt.show()



# Most people coming to ireland for
# Real estate, renting and business activities (Africa, North America, Oceania), 
# Manufacturing (Europe, South & Central America and Carribbean) and
# and Health and social work (Asia)
# (and wholesale)


# import OECD3
OECD3 = pd.read_csv('Data/OECD_3_Income_Tax_Married100_2Children.csv', sep = ',')


OECD3.head()
# want: percentage Average income tax rate (% gross wage earnings)


# check unique
print(OECD3['Unit'].unique())
print(OECD3['Household type'].unique())

['US Dollar' 'Percentage' 'Australian Dollar' 'Euro' 'Canadian Dollar'
 'Czech Koruna' 'Danish Krone' 'Forint' 'Iceland Krona' 'Yen' 'Won'
 'Mexican Peso' 'New Zealand Dollar' 'Norwegian Krone' 'Zloty'
 'Swedish Krona' 'Swiss Franc' 'Turkish Lira' 'Pound Sterling'
 'Chilean Peso' 'New Israeli Sheqel' nan 'Colombian Peso'
 'Costa Rican Colon']
['Two-earner married couple, one at 100% of average earnings and the other at 67%, with two children'
 'Single person at 167% of average earnings, without child'
 'Single person at 100% of average earnings, without child'
 'Single person at 67% of average earnings, with two children'
 'Two-earner married couple, one at 100% of average earnings and the other at 67%, without child'
 'One-earner married couple at 100% of average earnings, with two children'
 'Two-earner married couple, one at 100% of average earnings and the other at 100%, with two children'
 'Single person at 67% of average earnings, without child']


print(OECD3['Country'].unique())

['Australia' 'Austria' 'Belgium' 'Canada' 'Czechia' 'Denmark' 'Finland'
 'France' 'Germany' 'Greece' 'Hungary' 'Iceland' 'Ireland' 'Italy' 'Japan'
 'Korea' 'Luxembourg' 'Mexico' 'Netherlands' 'New Zealand' 'Norway'
 'Poland' 'Portugal' 'Slovak Republic' 'Spain' 'Sweden' 'Switzerland'
 'Türkiye' 'United Kingdom' 'United States' 'Chile' 'Estonia' 'Israel'
 'Slovenia' 'OECD - Average' 'Latvia' 'European Union 22 members in OECD'
 'Lithuania' 'Colombia' 'Costa Rica']


print(OECD3['Indicator'].unique())

['Total gross earnings before taxes in US dollars using PPP exchange rates'
 'Marginal tax wedge: Principal earner (% labour costs)'
 'Net income after taxes in national currency'
 'Net personal average tax rate (% gross wage earnings)'
 "Average rate of employees' social security contributions (% gross wage earnings)"
 'Increase in net income after an increase of 1 currency unit in gross wages'
 'Total gross labour costs before taxes in US dollars using PPP exchange rates'
 'Total gross earnings before taxes in national currency'
 "Average rate of income tax and employees' social security contributions (% gross wage earnings)"
 'Increase in net income after an increase of 1 currency unit in gross labour cost'
 'Net personal marginal tax rate: Principal earner (% gross wage earnings)'
 'Average income tax rate (% gross wage earnings)'
 'Net income after taxes in US dollars using PPP exchange rates'
 "Average rate of employer's social security contributions (% gross wage earnings)"
 'Average tax wedge (% labour costs)']


OECD3 = OECD3[OECD3['Country'].isin(["Ireland", 
                                     "European Union 22 members in OECD", 
                                     "OECD - Average", 
                                     "United States", 
                                     "United Kingdom"])]


OECD3 = OECD3[OECD3['Household type'].isin(["Single person at 100% of average earnings, without child", "Two-earner married couple, one at 100% of average earnings and the other at 67%, with two children"])]


OECD3 = OECD3[OECD3['Unit'].isin(["Percentage"])]


OECD3 = OECD3[OECD3['Indicator'].isin(["Average income tax rate (% gross wage earnings)"])]


# Select columns
OECD3 = OECD3[['Year', 'Country', 'Household type','Value']]


OECD3.head()


# Tax married couple at 100% earning w 2 children
sns.lineplot(data=OECD3[(OECD3['Household type'] == 'Two-earner married couple, one at 100% of average earnings and the other at 67%, with two children')],
              x='Year', 
              y='Value', 
              hue='Country')
plt.xlabel('Year')
plt.title('Income tax married couple at 100% earning w 2 children over time')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Tax single at 100% earning wo child over time
sns.lineplot(data=OECD3[(OECD3['Household type'] == 'Single person at 100% of average earnings, without child')],
              x='Year', 
              y='Value', 
              hue='Country')
plt.xlabel('Year')
plt.title('Income tax single at 100% earning wo child over time')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


OECD3_2 = OECD3[OECD3['Country'].isin(["Ireland", 
                                     "European Union 22 members in OECD", 
                                     "United States", 
                                     "United Kingdom"])]


OECD3_2['Household type'].replace({
    "Two-earner married couple, one at 100% of average earnings and the other at 67%, with two children": "Tax Married 100% w 2 Children",
    "Single person at 100% of average earnings, without child": "Tax Single 100% wo Child"
}, inplace=True)

C:\Users\joosl\AppData\Local\Temp\ipykernel_17284\2432190941.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  OECD3_2['Household type'].replace({


OECD3_2['Country'].replace({
    "European Union 22 members in OECD": "EU in OECD"
}, inplace=True)

C:\Users\joosl\AppData\Local\Temp\ipykernel_17284\4133383591.py:1: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  OECD3_2['Country'].replace({


OECD3_2.head()


#OECD3_2 transpose Component column
OECD3_2 = OECD3_2.pivot_table(index = ['Year'], columns = ['Country', 'Household type'], values = 'Value').reset_index()


OECD3_2.head()


OECD3_2.columns = ['_'.join(col).strip() for col in OECD3_2.columns.values]


# Change column names
OECD3_2 = OECD3_2.rename(columns = {'Year_':'Year'})


OECD3_2.head()


# import TE01
TE01 = pd.read_csv('Data/TE01_10Y_Bond_Yields.csv', sep = ';')


TE01.head()


# remove month column and change columns
TE01 = TE01.drop(columns = ['Month', 'Percentage Change MoM', 'Change MoM'])


# change , to . 
TE01['Ireland 10Y Bond Yield'] = TE01['Ireland 10Y Bond Yield'].str.replace(',', '.')


# columns as numeric
TE01['Ireland 10Y Bond Yield'] = pd.to_numeric(TE01['Ireland 10Y Bond Yield'])


# Take average per year
TE01 = TE01.groupby(['Year']).mean().reset_index()


# Change name 1st column
TE01.columns.name = 'Index'


TE01['Year'] = pd.to_numeric(TE01['Year'])


TE01.head()


# 10 Year Bond Yields Ireland over Time
TE01.plot(x = 'Year', 
y = 'Ireland 10Y Bond Yield', 
xlabel='Year', 
title='Population Changes in Ireland from 1987 to 2023')

<Axes: title={'center': 'Population Changes in Ireland from 1987 to 2023'}, xlabel='Year'>


# import WB 1
WB01 = pd.read_csv('Data/WB01_GDP_Growth_Annual.csv', sep = ';')


WB01.head()


# Graph year and GDP Growth
WB01.plot(x='Year', y='GDP Growth (annual %)', xlabel='Year', title='GDP Growth in Ireland from 1971 to 2020')

<Axes: title={'center': 'GDP Growth in Ireland from 1971 to 2020'}, xlabel='Year'>


# import WB 2
WB02 = pd.read_csv('Data/WB02_GDP_perCapita_Growth_Annual.csv', sep = ';')


WB02.tail()


# Graph year and GDP Growth
WB02.plot(x='Year', y='GDP per Capita Growth (Annual %)', xlabel='Year', title='GDP per Capita Growth in Ireland from 1971 to 2022')

<Axes: title={'center': 'GDP per Capita Growth in Ireland from 1971 to 2022'}, xlabel='Year'>


# import WB 3
WB03 = pd.read_csv('Data/WB03_Inflation_Consumer_Prices_Annual.csv', sep = ';')


WB03.tail()


# Graph year and GDP Growth
WB03.plot(x='Year', y='Inflation Consumer Prices (Annual %)', xlabel='Year', title='Inflation Consumer Prices in Ireland from 1960 to 2022')

<Axes: title={'center': 'Inflation Consumer Prices in Ireland from 1960 to 2022'}, xlabel='Year'>


# import WB 4
WB04 = pd.read_csv('Data/WB04_Inflation_GDP_Deflator_Annual.csv', sep = ';')


WB04.head()


# Graph year and GDP Growth
WB04.plot(x='Year', y='Inflation GDP Deflator (Annual %)', xlabel='Year', title='Inflation GDP Deflator in Ireland from 1971 to 2022')

<Axes: title={'center': 'Inflation GDP Deflator in Ireland from 1971 to 2022'}, xlabel='Year'>


# Combine WB01, WB02, WB03 and WB04 on Year including all years in WB03
WB = WB03.merge(WB01, on='Year', how='outer').merge(WB02, on='Year', how = 'outer').merge(WB04, on='Year', how = 'outer')
WB.head()


# graph
sns.lineplot(data=WB, 
             x='Year', y='GDP Growth (annual %)', label='GDP Growth (annual %)')
sns.lineplot(data=WB, 
             x='Year', y='GDP per Capita Growth (Annual %)', label='GDP per Capita Growth (Annual %)')
sns.lineplot(data=WB, 
             x='Year', y='Inflation Consumer Prices (Annual %)', label='Inflation Consumer Prices (Annual %)')
sns.lineplot(data=WB, 
             x='Year', y='Inflation GDP Deflator (Annual %)', label='Inflation GDP Deflator (Annual %)')
plt.xlabel('Year')
plt.title('Rates in Ireland')
plt.legend(title='Rates', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# graph
sns.lineplot(data=WB, 
             x='Year', y='GDP Growth (annual %)', label='GDP Growth (annual %)')
sns.lineplot(data=WB, 
             x='Year', y='Inflation Consumer Prices (Annual %)', label='Inflation Consumer Prices (Annual %)')
plt.xlabel('Year')
plt.ylabel('Percentages')
plt.title('Rates in Ireland')
plt.legend(title='Rates', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# import WB 5
WB05 = pd.read_csv('Data/WB05_Population_Country.csv', sep = ',')


WB05.head()


# check unique values
print(WB05['Series Name'].unique())

[' total' nan]


WB05 = WB05.drop(columns=['Series Name', 'Series Code', 'Country Code'])


# clean up column names
# Define a function to remove everything after the space
def remove_after_space(col):
    return col.split(' ')[0]

# Apply the function to all column names and rename the columns
WB05.rename(columns=remove_after_space, inplace=True)


# pivot columns for years
WB05 = WB05.melt(id_vars=['Country'], var_name='Year', value_name='Population')


print(WB05['Country'].unique())

['Afghanistan' 'Albania' 'Algeria' 'American Samoa' 'Andorra' 'Angola'
 'Antigua and Barbuda' 'Argentina' 'Armenia' 'Aruba' 'Australia' 'Austria'
 'Azerbaijan' nan 'Bahrain' 'Bangladesh' 'Barbados' 'Belarus' 'Belgium'
 'Belize' 'Benin' 'Bermuda' 'Bhutan' 'Bolivia' 'Bosnia and Herzegovina'
 'Botswana' 'Brazil' 'British Virgin Islands' 'Brunei Darussalam'
 'Bulgaria' 'Burkina Faso' 'Burundi' 'Cabo Verde' 'Cambodia' 'Cameroon'
 'Canada' 'Cayman Islands' 'Central African Republic' 'Chad'
 'Channel Islands' 'Chile' 'China' 'Colombia' 'Comoros' 'Costa Rica'
 "Cote d'Ivoire" 'Croatia' 'Cuba' 'Curacao' 'Cyprus' 'Czechia' 'Denmark'
 'Djibouti' 'Dominica' 'Dominican Republic' 'Ecuador' 'El Salvador'
 'Equatorial Guinea' 'Eritrea' 'Estonia' 'Eswatini' 'Ethiopia'
 'Faroe Islands' 'Fiji' 'Finland' 'France' 'French Polynesia' 'Gabon'
 'Georgia' 'Germany' 'Ghana' 'Gibraltar' 'Greece' 'Greenland' 'Grenada'
 'Guam' 'Guatemala' 'Guinea' 'Guinea-Bissau' 'Guyana' 'Haiti' 'Honduras'
 'Hungary' 'Iceland' 'India' 'Indonesia' 'Iraq' 'Ireland' 'Isle of Man'
 'Israel' 'Italy' 'Jamaica' 'Japan' 'Jordan' 'Kazakhstan' 'Kenya'
 'Kiribati' 'Kosovo' 'Kuwait' 'Kyrgyz Republic' 'Lao PDR' 'Latvia'
 'Lebanon' 'Lesotho' 'Liberia' 'Libya' 'Liechtenstein' 'Lithuania'
 'Luxembourg' 'Madagascar' 'Malawi' 'Malaysia' 'Maldives' 'Mali' 'Malta'
 'Marshall Islands' 'Mauritania' 'Mauritius' 'Mexico' 'Moldova' 'Monaco'
 'Mongolia' 'Montenegro' 'Morocco' 'Mozambique' 'Myanmar' 'Namibia'
 'Nauru' 'Nepal' 'Netherlands' 'New Caledonia' 'New Zealand' 'Nicaragua'
 'Niger' 'Nigeria' 'North Macedonia' 'Northern Mariana Islands' 'Norway'
 'Oman' 'Pakistan' 'Palau' 'Panama' 'Papua New Guinea' 'Paraguay' 'Peru'
 'Philippines' 'Poland' 'Portugal' 'Puerto Rico' 'Qatar' 'Romania'
 'Russian Federation' 'Rwanda' 'Samoa' 'San Marino'
 'Sao Tome and Principe' 'Saudi Arabia' 'Senegal' 'Serbia' 'Seychelles'
 'Sierra Leone' 'Singapore' 'Sint Maarten (Dutch part)' 'Slovak Republic'
 'Slovenia' 'Solomon Islands' 'Somalia' 'South Africa' 'South Sudan'
 'Spain' 'Sri Lanka' 'St. Kitts and Nevis' 'St. Lucia'
 'St. Martin (French part)' 'St. Vincent and the Grenadines' 'Sudan'
 'Suriname' 'Sweden' 'Switzerland' 'Syrian Arab Republic' 'Tajikistan'
 'Tanzania' 'Thailand' 'Timor-Leste' 'Togo' 'Tonga' 'Trinidad and Tobago'
 'Tunisia' 'Turkiye' 'Turkmenistan' 'Turks and Caicos Islands' 'Tuvalu'
 'Uganda' 'Ukraine' 'United Arab Emirates' 'United Kingdom'
 'United States' 'Uruguay' 'Uzbekistan' 'Vanuatu' 'Viet Nam'
 'Virgin Islands (U.S.)' 'West Bank and Gaza' 'Zambia' 'Zimbabwe'
 'Africa Eastern and Southern' 'Africa Western and Central' 'Arab World'
 'Caribbean small states' 'Central Europe and the Baltics'
 'Early-demographic dividend' 'East Asia & Pacific'
 'East Asia & Pacific (excluding high income)'
 'East Asia & Pacific (IDA & IBRD countries)' 'Euro area'
 'Europe & Central Asia' 'Europe & Central Asia (excluding high income)'
 'Europe & Central Asia (IDA & IBRD countries)' 'European Union'
 'Fragile and conflict affected situations'
 'Heavily indebted poor countries (HIPC)' 'High income' 'IBRD only'
 'IDA & IBRD total' 'IDA blend' 'IDA only' 'IDA total'
 'Late-demographic dividend' 'Latin America & Caribbean'
 'Latin America & Caribbean (excluding high income)'
 'Latin America & the Caribbean (IDA & IBRD countries)'
 'Least developed countries: UN classification' 'Low & middle income'
 'Low income' 'Lower middle income' 'Middle East & North Africa'
 'Middle East & North Africa (excluding high income)'
 'Middle East & North Africa (IDA & IBRD countries)' 'Middle income'
 'North America' 'Not classified' 'OECD members' 'Other small states'
 'Pacific island small states' 'Post-demographic dividend'
 'Pre-demographic dividend' 'Small states' 'South Asia'
 'South Asia (IDA & IBRD)' 'Sub-Saharan Africa'
 'Sub-Saharan Africa (excluding high income)'
 'Sub-Saharan Africa (IDA & IBRD countries)' 'Upper middle income' 'World']


WB05 = WB05[WB05['Country'].isin(['Ireland', 'United Kingdom', 'United States', 
'European Union', 'OECD members', 'South Asia',
'North America', 'East Asia & Pacific', 'India', 'China'])]

# South Asia: Afghanistan, Bangladesh, Bhutan, India, Maldives, Nepal, Pakistan, and Sri Lanka
# East Asia: Japan, North and South Korea, China, Mongolia, and Taiwan


WB05.tail()


# for .. put nothing
WB05['Population'] = WB05['Population'].str.replace('..', '')


# columns as numeric
WB05['Year'] = pd.to_numeric(WB05['Year'])
WB05['Population'] = pd.to_numeric(WB05['Population'])


# Population by Country
sns.lineplot(data=WB05,
              x='Year', 
              y='Population', 
              hue='Country')
plt.xlabel('Year')
plt.title('Population by Country over Time')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


WB05 = WB05[WB05['Country'].isin(['South Asia', 'East Asia & Pacific', 'United Kingdom','Ireland','European Union'])]


# Population by Country
sns.lineplot(data=WB05,
              x='Year', 
              y='Population', 
              hue='Country')
plt.xlabel('Year')
plt.title('Population by Country over Time')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Population by Country
sns.lineplot(data=WB05[(WB05['Country']=="Ireland")],
              x='Year', 
              y='Population', 
              hue='Country')
plt.xlabel('Year')
plt.title('Population by Country over Time')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# Population by Country
sns.lineplot(data=WB05[(WB05['Country']=="India")],
              x='Year', 
              y='Population', 
              hue='Country')
plt.xlabel('Year')
plt.title('Population by Country over Time')
plt.legend(title='Country', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.


#WB05 transpose Component column
WB05 = WB05.pivot_table(index = ['Year'], columns = 'Country', values = 'Population').reset_index()


WB05.tail()


for index, row in WB05.iterrows():
    Year = row['Year']
    East_Asia_Pacific = row['East Asia & Pacific']
    European_Union = row['European Union']
    Ireland = row['Ireland']
    South_Asia = row['South Asia']
    UK = row['United Kingdom']

    # Get Ireland values 
    Ireland_current_year = Ireland
    Ireland_previous_year = WB05.loc[index - 1, 'Ireland'] if index > 0 else None
    Ireland_percentage_change = (Ireland_current_year - Ireland_previous_year) / Ireland_previous_year * 100 if Ireland_previous_year else None

    # Get East Asia and Pacific values 
    East_Asia_Pacific_current_year = East_Asia_Pacific
    East_Asia_Pacific_previous_year = WB05.loc[index - 1, 'East Asia & Pacific'] if index > 0 else None
    East_Asia_Pacific_percentage_change = (East_Asia_Pacific_current_year - East_Asia_Pacific_previous_year) / East_Asia_Pacific_previous_year * 100 if East_Asia_Pacific_previous_year else None

    # Get European_Union values 
    European_Union_current_year = European_Union
    European_Union_previous_year = WB05.loc[index - 1, 'European Union'] if index > 0 else None
    European_Union_percentage_change = (European_Union_current_year - European_Union_previous_year) / European_Union_previous_year * 100 if European_Union_previous_year else None

    # Get South Asia values 
    South_Asia_current_year = South_Asia
    South_Asia_previous_year = WB05.loc[index - 1, 'South Asia'] if index > 0 else None
    South_Asia_percentage_change = (South_Asia_current_year - South_Asia_previous_year) / South_Asia_previous_year * 100 if South_Asia_previous_year else None

    # Get UK values 
    UK_current_year = UK
    UK_previous_year = WB05.loc[index - 1, 'United Kingdom'] if index > 0 else None
    UK_percentage_change = (UK_current_year - UK_previous_year) / UK_previous_year * 100 if UK_previous_year else None

    # Store to dataframe
    WB05.loc[index, 'Ireland Percentage Change'] = Ireland_percentage_change
    WB05.loc[index, 'East Asia & Pacific Percentage Change'] = East_Asia_Pacific_percentage_change
    WB05.loc[index, 'European Union Percentage Change'] = European_Union_percentage_change
    WB05.loc[index, 'South Asia Percentage Change'] = South_Asia_percentage_change
    WB05.loc[index, 'UK Percentage Change'] = UK_percentage_change


# graph
sns.lineplot(data=WB05, 
             x='Year', y='Ireland Percentage Change', label='Ireland Percentage Change')
sns.lineplot(data=WB05, 
             x='Year', y='East Asia & Pacific Percentage Change', label='East Asia & Pacific Percentage Change')
sns.lineplot(data=WB05, 
             x='Year', y='European Union Percentage Change', label='European Union Percentage Change')             
sns.lineplot(data=WB05, 
             x='Year', y='South Asia Percentage Change', label='South Asia Percentage Change')
sns.lineplot(data=WB05, 
             x='Year', y='UK Percentage Change', label='UK Percentage Change')
plt.xlabel('Year')
plt.title('Population Changes')
plt.legend(title='Geographic Location', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# combine WB with PEA15
WB_PEA15 = PEA15.merge(WB, on='Year', how='outer')
WB_PEA15.head()


WB_PEA15 = WB_PEA15.drop(['Natural increase', 'Population', 'Population change'], axis=1)
WB_PEA15.head()


OECD1.head()


# Filter OECD1
OECD1_2 = OECD1[OECD1['Country'].isin(['United Kingdom', 'Ireland'])]
OECD1_2 = OECD1_2[OECD1_2['Gender'].isin(['Total', 'Women'])]
OECD1_2.head()


#OECD1_2 transpose Component column
OECD1_2 = OECD1_2.pivot_table(index = ['Year'], columns = ['Country', 'Gender', 'Place_of_Birth'], values = ['Participation rate', 'Unemployment rate']).reset_index()


OECD1_2.head()


OECD1_2.columns = ['_'.join(col).strip() for col in OECD1_2.columns.values]


# Change column names
OECD1_2 = OECD1_2.rename(columns = {'Year___':'Year'})


OECD1_2.tail(n=10)


# combine WB_PEA15 with Foreign Irish Employment
WB_PEA15_OECD1 = WB_PEA15.merge(OECD1_2, on='Year', how='outer')
WB_PEA15_OECD1.tail()


WB_PEA15_OECD1.tail(n=10)


# create dataframe for Employment rates Natives outside Ireland
OECD1_OtherCountries = OECD1[(OECD1['Country'] != 'OECD - Total') & 
                             (OECD1['Country'] != 'Ireland') & 
                             (OECD1['Place_of_Birth'] == 'Native-born') & 
                             (OECD1['Gender'] == 'Total')]

# check unique values for country (no ireland or aggregates)
print(OECD1_OtherCountries['Country'].unique())

# drop irrelevant columns
OECD1_OtherCountries = OECD1_OtherCountries.drop(['Place_of_Birth', 'Gender', 'Country'], axis=1)

# aggregate data based on year for employment rate, participation rate and unemployment rate and not taking into account the country
OECD1_OtherCountries = OECD1_OtherCountries.groupby('Year').mean().reset_index()

# change column names Native_Irish_Employment
OECD1_OtherCountries = OECD1_OtherCountries.rename(columns = {'Employment rate': 'Avg Employment Natives outside Ireland', 
                                                              'Participation rate': 'Avg Participation Natives outside Ireland',
                                                              'Unemployment rate': 'Avg Unemployment Natives outside Ireland'})

# check if worked
OECD1_OtherCountries.head()

['Australia' 'Austria' 'Belgium' 'Denmark' 'Estonia' 'Finland' 'France'
 'Germany' 'Greece' 'Hungary' 'Iceland' 'Italy' 'Luxembourg' 'Netherlands'
 'Norway' 'Portugal' 'Slovenia' 'Spain' 'Sweden' 'United Kingdom'
 'United States' 'Czechia' 'New Zealand' 'Poland' 'Slovak Republic'
 'Switzerland' 'Mexico' 'Chile' 'Canada' 'Türkiye' 'Israel' 'Latvia'
 'Lithuania']


# combine WB_PEA15 with Foreign Irish Employment
WB_PEA15_OECD1 = WB_PEA15_OECD1.merge(OECD1_OtherCountries, on='Year', how='outer')
WB_PEA15_OECD1.tail()


# create dataframe for Employment rates Natives outside Ireland (Women)
OECD1_OtherCountriesWomen = OECD1[(OECD1['Country'] != 'OECD - Total') & 
                             (OECD1['Country'] != 'Ireland') & 
                             (OECD1['Place_of_Birth'] == 'Native-born') & 
                             (OECD1['Gender'] == 'Women')]

# check unique values for country (no ireland or aggregates)
print(OECD1_OtherCountriesWomen['Country'].unique())

# drop irrelevant columns
OECD1_OtherCountriesWomen = OECD1_OtherCountriesWomen.drop(['Place_of_Birth', 'Gender', 'Country'], axis=1)

# aggregate data based on year for employment rate, participation rate and unemployment rate and not taking into account the country
OECD1_OtherCountriesWomen = OECD1_OtherCountriesWomen.groupby('Year').mean().reset_index()

# change column names Native_Irish_Employment
OECD1_OtherCountriesWomen = OECD1_OtherCountriesWomen.rename(columns = {'Employment rate': 'Avg Employment Native Women outside Ireland', 
                                                              'Participation rate': 'Avg Participation Native Women outside Ireland',
                                                              'Unemployment rate': 'Avg Unemployment Native Women outside Ireland'})

# check if worked
OECD1_OtherCountriesWomen.head()

['Australia' 'Austria' 'Belgium' 'Denmark' 'Estonia' 'Finland' 'France'
 'Germany' 'Greece' 'Hungary' 'Iceland' 'Italy' 'Luxembourg' 'Netherlands'
 'Norway' 'Portugal' 'Slovenia' 'Spain' 'Sweden' 'United Kingdom'
 'United States' 'Czechia' 'New Zealand' 'Poland' 'Slovak Republic'
 'Switzerland' 'Mexico' 'Chile' 'Canada' 'Türkiye' 'Israel' 'Latvia'
 'Lithuania']


# combine WB_PEA15 with Foreign Irish Employment
WB_PEA15_OECD1 = WB_PEA15_OECD1.merge(OECD1_OtherCountriesWomen, on='Year', how='outer')
WB_PEA15_OECD1.tail()


# drop irrelevant columns and check data
PEA03_edit = PEA03.drop(['Sex'], axis=1)
PEA03_edit.head()


# filter on 15-24 age group
PEA03_1524 = PEA03_edit[(PEA03_edit['Age Group'] == '15 - 24 years')]
PEA03_1524 = PEA03_1524.drop(['Age Group'], axis=1)

# aggregate data based on year for emigrants, immigrants, net migration
PEA03_1524 = PEA03_1524.groupby('Year').mean().reset_index()

# change column names Native_Irish_Employment
PEA03_1524 = PEA03_1524.rename(columns = {'Emigrants': 'Emigrants 15-24 years',
                                          'Immigrants': 'Immigrants 15-24 years',
                                          'Net migration': 'Net migration 15-24 years'})


PEA03_1524.head()


# combine WB_PEA15_OECD1 with PEA03_1524
WB_PEA15_OECD1_PEA03 = WB_PEA15_OECD1.merge(PEA03_1524, on='Year', how='outer')
WB_PEA15_OECD1_PEA03.tail()


# filter on 25-44 age group
PEA03_2544 = PEA03_edit[(PEA03_edit['Age Group'] == '25 - 44 years')]
PEA03_2544 = PEA03_2544.drop(['Age Group'], axis=1)

# aggregate data based on year for emigrants, immigrants, net migration
PEA03_2544 = PEA03_2544.groupby('Year').mean().reset_index()

# change column names Native_Irish_Employment
PEA03_2544 = PEA03_2544.rename(columns = {'Emigrants': 'Emigrants 25-44 years',
                                          'Immigrants': 'Immigrants 25-44 years',
                                          'Net migration': 'Net migration 25-44 years'})


PEA03_2544.head()


# combine WB_PEA15_OECD1_PEA03 with PEA03_2544
WB_PEA15_OECD1_PEA03 = WB_PEA15_OECD1_PEA03.merge(PEA03_2544, on='Year', how='outer')
WB_PEA15_OECD1_PEA03.tail()


print(CPM01['Commodity Group'].unique())

['All items' 'Alcoholic beverages and tobacco' 'Clothing and footwear'
 'Communications' 'Education' 'Food and non-alcoholic beverages'
 'Furnishings, household equipment and routine household maintenance'
 'Health' 'Housing, water, electricity, gas and other fuels'
 'Miscellaneous goods and services' 'Recreation and culture'
 'Restaurants and hotels' 'Transport']


CPM01.head()


# filter on housing
CPM01_Housing = CPM01[(CPM01['Commodity Group'] == 'Housing, water, electricity, gas and other fuels')]

CPM01_Housing = CPM01_Housing[['Year', 'Percentage Change over 12 months for Consumer Price Index']]

# change column name CPM01_Housing
CPM01_Housing = CPM01_Housing.rename(columns = {'Percentage Change over 12 months for Consumer Price Index': 'Housing_CPI'})

CPM01_Housing.head()


# combine WB_PEA15_OECD1_PEA03 with CPM01_Housing
WB_PEA15_OECD1_PEA03_CPM01 = WB_PEA15_OECD1_PEA03.merge(CPM01_Housing, on='Year', how='outer')
WB_PEA15_OECD1_PEA03_CPM01.tail()


# filter on food and NADrinks
CPM01_Food_Drinks = CPM01[(CPM01['Commodity Group'] == 'Food and non-alcoholic beverages')]

CPM01_Food_Drinks = CPM01_Food_Drinks[['Year', 'Percentage Change over 12 months for Consumer Price Index']]

# change column name CPM01_Food_Drinks
CPM01_Food_Drinks = CPM01_Food_Drinks.rename(columns = {'Percentage Change over 12 months for Consumer Price Index': 'Food_Drinks_CPI'})

CPM01_Food_Drinks.head()


# combine WB_PEA15_OECD1_PEA03_CPM01 with CPM01_Food_Drinks
WB_PEA15_OECD1_PEA03_CPM01 = WB_PEA15_OECD1_PEA03_CPM01.merge(CPM01_Food_Drinks, on='Year', how='outer')
WB_PEA15_OECD1_PEA03_CPM01.tail()


# filter on housing
CPM01_General = CPM01[(CPM01['Commodity Group'] == 'All items')]

CPM01_General = CPM01_General[['Year', 'Percentage Change over 12 months for Consumer Price Index']]

# change column name CPM01_General
CPM01_General = CPM01_General.rename(columns = {'Percentage Change over 12 months for Consumer Price Index': 'General_CPI'})

CPM01_General.tail()


# combine WB_PEA15_OECD1_PEA03_CPM01 with CPM01_General
WB_PEA15_OECD1_PEA03_CPM01 = WB_PEA15_OECD1_PEA03_CPM01.merge(CPM01_General, on='Year', how='outer')
WB_PEA15_OECD1_PEA03_CPM01.tail()


# select columns for TSA02
TSA02_edit = TSA02[['Year', 
                    'Trade Price Index Exports (Base 2010)', 
                    'Trade Price Index Imports (Base 2010)', 
                    'Trade Volume Index Exports (Base 2010)', 
                    'Trade Volume Index Imports (Base 2010)']]
TSA02_edit.head()


WB_PEA15_OECD1_PEA03_CPM01.head()


# combine WB_PEA15_OECD1_PEA03 with TSA02_edit
Combined_All_w_CSO = WB_PEA15_OECD1_PEA03_CPM01.merge(TSA02_edit, on='Year', how='inner')
Combined_All_w_CSO.head()


Combined_All_w_CSO = Combined_All_w_CSO.merge(BPA33_2, on='Year', how='outer')
Combined_All_w_CSO.tail()


# combine Combined_All_w_CSO with GFA01
Combined_All_w_CSO = Combined_All_w_CSO.merge(GFA01, on='Year', how='outer')
Combined_All_w_CSO.tail()


# combine Combined_All_w_CSO with BRA08
Combined_All_w_CSO = Combined_All_w_CSO.merge(BRA08, on='Year', how='outer')
Combined_All_w_CSO.tail()


# combine Combined_All_w_CSO with BRA08
Combined_All_w_CSO = Combined_All_w_CSO.merge(OECD3_2, on='Year', how='outer')
Combined_All_w_CSO.tail()


# combine Combined_All_w_CSO with WB05
Combined_All_w_CSO_WB = Combined_All_w_CSO.merge(WB05, on='Year', how='outer')
Combined_All_w_CSO_WB.tail()


# immigrants, emigrants and net migration divided by Ireland
Combined_All_w_CSO_WB['Immigrant Percentage of Population'] = Combined_All_w_CSO_WB['Immigrants'] / Combined_All_w_CSO_WB['Ireland'] *100
Combined_All_w_CSO_WB['Emigrant Percentage of Population'] = Combined_All_w_CSO_WB['Emigrants'] / Combined_All_w_CSO_WB['Ireland'] * 100
Combined_All_w_CSO_WB['Migration Percentage of Population'] = Combined_All_w_CSO_WB['Net migration'] / Combined_All_w_CSO_WB['Ireland'] *100


Combined_All_w_CSO_WB.tail()


# graph
sns.lineplot(data=Combined_All_w_CSO_WB, 
             x='Year', y='Emigrant Percentage of Population', label='Emigrant Percentage of Population')
sns.lineplot(data=Combined_All_w_CSO_WB, 
             x='Year', y='Immigrant Percentage of Population', label='Immigrant Percentage of Population')
sns.lineplot(data=Combined_All_w_CSO_WB, 
             x='Year', y='Migration Percentage of Population', label='Migration Percentage of Population')
plt.xlabel('Year')
plt.title('Population Changes')
plt.legend(title='Ireland', bbox_to_anchor=(1.05, 1), loc='upper left')
plt.show()


# combine Combined_All_w_CSO with TE01
Combined_All_w_CSO_WB_TE = Combined_All_w_CSO_WB.merge(TE01, on='Year', how='outer')
Combined_All_w_CSO_WB_TE.tail()


Combined_All_w_CSO_WB_TE.head()


# combine Combined_All_w_CSO with TE01
Final = Combined_All_w_CSO_WB_TE.merge(PEA18_2, on='Year', how='outer')
Final.head()


Final.head(n=25)


Final.tail(n=50)


print(Final.columns)

Index(['Year', 'Annual births', 'Annual deaths', 'Emigrants', 'Immigrants',
       'Net migration', 'Inflation Consumer Prices (Annual %)',
       'GDP Growth (annual %)', 'GDP per Capita Growth (Annual %)',
       'Inflation GDP Deflator (Annual %)',
       'Participation rate_Ireland_Total_Foreign-born',
       'Participation rate_Ireland_Total_Native-born',
       'Participation rate_Ireland_Women_Foreign-born',
       'Participation rate_Ireland_Women_Native-born',
       'Participation rate_United Kingdom_Total_Foreign-born',
       'Participation rate_United Kingdom_Total_Native-born',
       'Participation rate_United Kingdom_Women_Foreign-born',
       'Participation rate_United Kingdom_Women_Native-born',
       'Unemployment rate_Ireland_Total_Foreign-born',
       'Unemployment rate_Ireland_Total_Native-born',
       'Unemployment rate_Ireland_Women_Foreign-born',
       'Unemployment rate_Ireland_Women_Native-born',
       'Unemployment rate_United Kingdom_Total_Foreign-born',
       'Unemployment rate_United Kingdom_Total_Native-born',
       'Unemployment rate_United Kingdom_Women_Foreign-born',
       'Unemployment rate_United Kingdom_Women_Native-born',
       'Avg Employment Natives outside Ireland',
       'Avg Participation Natives outside Ireland',
       'Avg Unemployment Natives outside Ireland',
       'Avg Employment Native Women outside Ireland',
       'Avg Participation Native Women outside Ireland',
       'Avg Unemployment Native Women outside Ireland',
       'Emigrants 15-24 years', 'Immigrants 15-24 years',
       'Net migration 15-24 years', 'Emigrants 25-44 years',
       'Immigrants 25-44 years', 'Net migration 25-44 years', 'Housing_CPI',
       'Food_Drinks_CPI', 'General_CPI',
       'Trade Price Index Exports (Base 2010)',
       'Trade Price Index Imports (Base 2010)',
       'Trade Volume Index Exports (Base 2010)',
       'Trade Volume Index Imports (Base 2010)',
       'Direct Investment  Flows Abroad_All countries and international organisations',
       'Direct Investment  Flows Abroad_Asia (6)',
       'Direct Investment  Flows Abroad_EU28',
       'Direct Investment  Flows Abroad_United States',
       'Direct Investment  Flows in Ireland_All countries and international organisations',
       'Direct Investment  Flows in Ireland_Asia (6)',
       'Direct Investment  Flows in Ireland_EU28',
       'Direct Investment  Flows in Ireland_United States',
       'General Surplus/Deficit (Million Euros)',
       'General Expenditure (Million Euros)',
       'Net Acquisition Financial Assets (Million Euros)',
       'Enterprises_Ireland_250+', 'Enterprises_Dublin_250+',
       'Enterprises_Ireland', 'Enterprises_Dublin',
       'EU in OECD_Tax Married 100% w 2 Children',
       'EU in OECD_Tax Single 100% wo Child',
       'Ireland_Tax Married 100% w 2 Children',
       'Ireland_Tax Single 100% wo Child',
       'United Kingdom_Tax Married 100% w 2 Children',
       'United Kingdom_Tax Single 100% wo Child',
       'United States_Tax Married 100% w 2 Children',
       'United States_Tax Single 100% wo Child', 'East Asia & Pacific',
       'European Union', 'Ireland', 'South Asia', 'United Kingdom',
       'Ireland Percentage Change', 'East Asia & Pacific Percentage Change',
       'European Union Percentage Change', 'South Asia Percentage Change',
       'UK Percentage Change', 'Immigrant Percentage of Population',
       'Emigrant Percentage of Population',
       'Migration Percentage of Population', 'Ireland 10Y Bond Yield',
       'Emigrants_All countries', 'Emigrants_EU14 excl Irl (UK & Ireland)',
       'Emigrants_EU15 to EU27 (accession countries joined post 2004)',
       'Emigrants_Other countries (23)', 'Emigrants_United Kingdom (1)',
       'Emigrants_United States', 'Immigrants_All countries',
       'Immigrants_EU14 excl Irl (UK & Ireland)',
       'Immigrants_EU15 to EU27 (accession countries joined post 2004)',
       'Immigrants_Other countries (23)', 'Immigrants_United Kingdom (1)',
       'Immigrants_United States', 'Net migration_All countries',
       'Net migration_EU14 excl Irl (UK & Ireland)',
       'Net migration_EU15 to EU27 (accession countries joined post 2004)',
       'Net migration_Other countries (23)',
       'Net migration_United Kingdom (1)', 'Net migration_United States'],
      dtype='object')


Final = Final.rename(columns = {'East Asia & Pacific': 'Population East Asia & Pacific', 'European Union': 'Population EU', 'Ireland': 'Population Ireland', 'South Asia':'Population South Asia', 'United Kingdom': 'Population UK'})


# correlatioon matrix Combined_All_w_CSO_WB_TE
corr = Final.corr()

# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corr, dtype=bool))

# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11, 9))


# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap='vlag', vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})

plt.show()


# correlatioon matrix Combined_All_w_CSO_WB_TE
corr = Combined_All_w_CSO_WB_TE.corr()

# Generate a mask for the upper triangle
mask = np.triu(np.ones_like(corr, dtype=bool))

# Set up the matplotlib figure
f, ax = plt.subplots(figsize=(11, 9))


# Draw the heatmap with the mask and correct aspect ratio
sns.heatmap(corr, mask=mask, cmap='vlag', vmax=.3, center=0,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})

plt.show()


# save dataset to excel
Final.to_excel('Data/FinalData.xlsx', index=False)

	STATISTIC Label	Year	Component	UNIT	VALUE
579	Annual Population Change	2023	Immigrants	Thousand	141.6
580	Annual Population Change	2023	Emigrants	Thousand	64.0
581	Annual Population Change	2023	Net migration	Thousand	77.6
582	Annual Population Change	2023	Population change	Thousand	97.6
583	Annual Population Change	2023	Population	Thousand	5281.6

Index	Year	Annual births	Annual deaths	Emigrants	Immigrants	Natural increase	Net migration	Population	Population change
0	1951	NaN	NaN	NaN	NaN	26600.0	-35000.0	2960600.0	-8400.0
1	1952	NaN	NaN	NaN	NaN	27300.0	-35000.0	2952900.0	-7700.0
2	1953	NaN	NaN	NaN	NaN	29100.0	-33000.0	2949000.0	-3900.0
3	1954	NaN	NaN	NaN	NaN	28200.0	-36000.0	2941200.0	-7800.0
4	1955	NaN	NaN	NaN	NaN	24700.0	-45000.0	2920900.0	-20300.0
5	1956	NaN	NaN	NaN	NaN	25600.0	-48000.0	2898500.0	-22400.0
6	1957	NaN	NaN	NaN	NaN	27800.0	-41000.0	2885300.0	-13200.0
7	1958	NaN	NaN	NaN	NaN	25400.0	-58000.0	2852700.0	-32600.0

Index	Year	Age Group	Sex	Emigrants: All destinations	Immigrants: All origins	Net migration
0	1987	0 - 14 years	Both sexes	2800.0	3100.0	NaN
1	1987	0 - 14 years	Female	1300.0	1800.0	NaN
2	1987	0 - 14 years	Male	1400.0	1300.0	NaN
3	1987	15 - 24 years	Both sexes	24000.0	5100.0	NaN
4	1987	15 - 24 years	Female	11700.0	3200.0	NaN
5	1987	15 - 24 years	Male	12300.0	1900.0	NaN
6	1987	25 - 44 years	Both sexes	11800.0	6100.0	NaN
7	1987	25 - 44 years	Female	4900.0	2700.0	NaN
8	1987	25 - 44 years	Male	6900.0	3400.0	NaN
9	1987	45 - 64 years	Both sexes	1700.0	1800.0	NaN
10	1987	45 - 64 years	Female	700.0	800.0	NaN
11	1987	45 - 64 years	Male	1000.0	1000.0	NaN
12	1987	65 years and over	Both sexes	0.0	1100.0	NaN
13	1987	65 years and over	Female	0.0	600.0	NaN
14	1987	65 years and over	Male	0.0	500.0	NaN
15	1987	All ages	Both sexes	40200.0	17200.0	-23000.0
16	1988	0 - 14 years	Both sexes	8300.0	3000.0	NaN
17	1988	0 - 14 years	Female	4000.0	1500.0	NaN
18	1988	0 - 14 years	Male	4300.0	1500.0	NaN
19	1988	15 - 24 years	Both sexes	31200.0	5400.0	NaN
20	1988	15 - 24 years	Female	14300.0	3000.0	NaN
21	1988	15 - 24 years	Male	17000.0	2400.0	NaN
22	1988	25 - 44 years	Both sexes	18300.0	7200.0	NaN
23	1988	25 - 44 years	Female	7300.0	3200.0	NaN
24	1988	25 - 44 years	Male	11000.0	4000.0	NaN

Index	Year	Country	Sex	Emigrants	Immigrants	Net migration
672	2022	EU15 to EU27 (accession countries joined post ...	Both sexes	5000.0	16800.0	11800.0
675	2022	Other countries (23)	Both sexes	10700.0	60500.0	49800.0
678	2022	United Kingdom (1)	Both sexes	13400.0	12800.0	-600.0
681	2022	United States	Both sexes	6300.0	4300.0	-2000.0
684	2023	All countries	Both sexes	64000.0	141600.0	77700.0
693	2023	EU14 excl Irl (UK & Ireland)	Both sexes	13900.0	17700.0	3800.0
696	2023	EU15 to EU27 (accession countries joined post ...	Both sexes	8800.0	8900.0	100.0
699	2023	Other countries (23)	Both sexes	11700.0	75800.0	64100.0
702	2023	United Kingdom (1)	Both sexes	14600.0	18400.0	3800.0
705	2023	United States	Both sexes	4900.0	5900.0	1000.0

	Year	Emigrants_All countries	Emigrants_EU14 excl Irl (UK & Ireland)	Emigrants_EU15 to EU27 (accession countries joined post 2004)	Emigrants_Other countries (23)	Emigrants_United Kingdom (1)	Emigrants_United States	Immigrants_All countries	Immigrants_EU14 excl Irl (UK & Ireland)	Immigrants_EU15 to EU27 (accession countries joined post 2004)	Immigrants_Other countries (23)	Immigrants_United Kingdom (1)	Immigrants_United States	Net migration_All countries	Net migration_EU14 excl Irl (UK & Ireland)	Net migration_EU15 to EU27 (accession countries joined post 2004)	Net migration_Other countries (23)	Net migration_United Kingdom (1)	Net migration_United States
27	2014	75000.0	14800.0	8000.0	16000.0	16400.0	6300.0	66500.0	11700.0	10800.0	18600.0	13500.0	4400.0	-8500.0	-3200.0	2800.0	2600.0	-3000.0	-1900.0
28	2015	70000.0	13300.0	5900.0	15900.0	16600.0	5100.0	75900.0	11900.0	11900.0	22800.0	16400.0	4400.0	5900.0	-1400.0	6000.0	6900.0	-200.0	-700.0
29	2016	66200.0	13900.0	9300.0	14200.0	14400.0	5700.0	82300.0	14700.0	11900.0	22200.0	18200.0	5300.0	16200.0	900.0	2700.0	8000.0	3800.0	-400.0
30	2017	56100.0	11200.0	8400.0	12700.0	10500.0	5600.0	95300.0	17200.0	13800.0	25600.0	21100.0	6100.0	39200.0	6000.0	5400.0	12900.0	10600.0	500.0
31	2018	51600.0	11300.0	6000.0	10800.0	10500.0	5300.0	96000.0	15900.0	11600.0	29100.0	21300.0	7800.0	44400.0	4600.0	5600.0	18300.0	10800.0	2500.0
32	2019	53100.0	10100.0	3700.0	12100.0	11200.0	5100.0	97100.0	15500.0	10000.0	34400.0	21600.0	6700.0	44000.0	5400.0	6300.0	22300.0	10400.0	1600.0
33	2020	50900.0	10700.0	7900.0	8700.0	9700.0	5300.0	95600.0	15500.0	9200.0	36400.0	17600.0	6000.0	44700.0	4800.0	1300.0	27700.0	7900.0	700.0
34	2021	52300.0	11100.0	4400.0	12500.0	17600.0	2400.0	74100.0	7900.0	8700.0	18400.0	21700.0	5500.0	21800.0	-3200.0	4300.0	5900.0	4100.0	3100.0
35	2022	56100.0	12800.0	5000.0	10700.0	13400.0	6300.0	107800.0	8100.0	16800.0	60500.0	12800.0	4300.0	51700.0	-4700.0	11800.0	49800.0	-600.0	-2000.0
36	2023	64000.0	13900.0	8800.0	11700.0	14600.0	4900.0	141600.0	17700.0	8900.0	75800.0	18400.0	5900.0	77700.0	3800.0	100.0	64100.0	3800.0	1000.0

	STATISTIC Label	Year	Age Group	Sex	Inward or Outward Flow	UNIT	VALUE
0	Estimated Migration (Persons in April)	1987	All ages	Both sexes	Net migration	Thousand	-23.0
1	Estimated Migration (Persons in April)	1987	All ages	Both sexes	Emigrants: All destinations	Thousand	40.2
2	Estimated Migration (Persons in April)	1987	All ages	Both sexes	Immigrants: All origins	Thousand	17.2
3	Estimated Migration (Persons in April)	1987	All ages	Male	Net migration	Thousand	NaN
4	Estimated Migration (Persons in April)	1987	All ages	Male	Emigrants: All destinations	Thousand	NaN

	Statistic Label	Year	Sex	Citizenship	UNIT	VALUE
0	Estimated Emigration (Persons in April)	2006	Both sexes	All Countries	Thousand	36.0
1	Estimated Emigration (Persons in April)	2006	Both sexes	EU14 excl Irl (countries in the EU pre 2004 ex...	Thousand	5.1
2	Estimated Emigration (Persons in April)	2006	Both sexes	EU15 to EU27 (accession countries joined post ...	Thousand	7.2
3	Estimated Emigration (Persons in April)	2006	Both sexes	United Kingdom	Thousand	2.2
4	Estimated Emigration (Persons in April)	2006	Both sexes	Ireland	Thousand	15.3

	Statistic Label	Year	Sex	Citizenship	UNIT	VALUE
0	Estimated Immigration (Persons in April)	1996	Both sexes	All Countries	Thousand	39.2
1	Estimated Immigration (Persons in April)	1996	Both sexes	EU14 excl Irl (countries in the EU pre 2004 ex...	Thousand	5.0
2	Estimated Immigration (Persons in April)	1996	Both sexes	EU15 to EU27 (accession countries joined post ...	Thousand	NaN
3	Estimated Immigration (Persons in April)	1996	Both sexes	United Kingdom	Thousand	8.3
4	Estimated Immigration (Persons in April)	1996	Both sexes	Ireland	Thousand	17.7

	Statistic Label	Month	Commodity Group	UNIT	VALUE
0	Consumer Price Index (Base Dec 2023=100)	1975 November	All items	Base Dec 2023=100	12.3
1	Consumer Price Index (Base Dec 2023=100)	1975 November	Food and non-alcoholic beverages	Base Dec 2023=100	NaN
2	Consumer Price Index (Base Dec 2023=100)	1975 November	Alcoholic beverages and tobacco	Base Dec 2023=100	NaN
3	Consumer Price Index (Base Dec 2023=100)	1975 November	Clothing and footwear	Base Dec 2023=100	NaN
4	Consumer Price Index (Base Dec 2023=100)	1975 November	Housing, water, electricity, gas and other fuels	Base Dec 2023=100	NaN

Index	Year	Commodity Group	Consumer Price Index (Base Dec 2001=100)	Consumer Price Index (Base Dec 2006=100)	Consumer Price Index (Base Dec 2011=100)	Consumer Price Index (Base Dec 2016=100)	Consumer Price Index (Base Dec 2023=100)	Consumer Price Index (Base Nov 1996=100)	Percentage Change over 1 month for Consumer Price Index	Percentage Change over 12 months for Consumer Price Index
0	1975	All items	18.600	15.700	15.100	14.900	12.300	21.900	0.000000	NaN
1	1976	All items	21.250	18.000	17.250	17.025	14.050	25.100	1.583333	20.300
2	1977	All items	24.125	20.425	19.575	19.325	15.975	28.525	0.875000	13.800
3	1978	All items	26.000	22.000	21.075	20.825	17.200	30.675	0.641667	7.650
4	1979	All items	29.400	24.925	23.850	23.600	19.450	34.750	1.241667	13.025

	Statistic Label	Year	State	UNIT	VALUE
0	Trade Price Index for Imports	1930	State	Base 2010=100	3.9
1	Trade Price Index for Imports	1931	State	Base 2010=100	3.3
2	Trade Price Index for Imports	1932	State	Base 2010=100	3.2
3	Trade Price Index for Imports	1933	State	Base 2010=100	3.0
4	Trade Price Index for Imports	1934	State	Base 2010=100	3.0

Index	Year	Terms of Trade Index (Base 2010)	Trade Price Index Exports (Base 2010)	Trade Price Index Imports (Base 2010)	Trade Volume Index Exports (Base 2010)	Trade Volume Index Imports (Base 2010)
0	1930	101.4	4.0	3.9	1.6	3.8
1	1931	106.9	3.6	3.3	1.4	3.9
2	1932	94.8	3.1	3.2	1.2	3.4
3	1933	87.2	2.6	3.0	1.0	3.2
4	1934	80.5	2.4	3.0	1.0	3.4

Index	Year	Terms of Trade Index (Base 2010)	Trade Price Index Exports (Base 2010)	Trade Price Index Imports (Base 2010)	Trade Volume Index Exports (Base 2010)	Trade Volume Index Imports (Base 2010)
88	2018	99.6	99.1	99.5	156.2	189.2
89	2019	103.0	99.5	96.6	168.5	192.7
90	2020	103.0	94.4	91.7	188.6	194.4
91	2021	96.4	95.0	98.6	191.3	214.5
92	2022	93.0	98.0	105.4	233.9	275.2

	Statistic Label	Year	Geographic Location	Type of Investment	UNIT	VALUE
0	Direct Investment Flows Abroad	2012	All countries and international organisations	All Direct Investment	Euro Million	17548.0
1	Direct Investment Flows Abroad	2012	United States	All Direct Investment	Euro Million	6734.0
2	Direct Investment Flows Abroad	2012	Asia (6)	All Direct Investment	Euro Million	920.0
3	Direct Investment Flows Abroad	2012	EU28	All Direct Investment	Euro Million	-278.0
4	Direct Investment Flows Abroad	2012	Europe (3)	All Direct Investment	Euro Million	4867.0

Statistic Label	Year	Direct Investment Flows Abroad				Direct Investment Flows in Ireland
Geographic Location		All countries and international organisations	Asia (6)	EU28	United States	All countries and international organisations	Asia (6)	EU28	United States
0	2012	17548.0	920.0	-278.0	6734.0	38046.0	4331.0	24122.0	3757.0
1	2013	21969.0	508.0	21057.0	109.0	38107.0	-952.0	22292.0	9615.0
2	2014	31038.0	1021.0	39875.0	-4107.0	36317.0	-3802.0	19555.0	6893.0
3	2015	151851.0	9304.0	66665.0	62870.0	196363.0	7702.0	48943.0	76457.0
4	2016	27180.0	-1815.0	6500.0	7056.0	35607.0	-11676.0	-25915.0	-55433.0

Index	Year	General Surplus/Deficit (Million Euros)	General Expenditure (Million Euros)	General Revenue (Million Euros)	Net Operating Balance (Million Euros)
0	1995	-1137	22096	20959	-857
1	1996	-119	23192	23074	288
2	1997	953	25049	26002	1536
3	1998	1666	27488	29154	2503
4	1999	3282	30165	33447	4614

Index	Year	General Surplus/Deficit (Million Euros)	General Expenditure (Million Euros)	Net Acquisition Financial Assets (Million Euros)
24	2019	1695	86604	-1107
25	2020	-18704	101989	-1118
26	2021	-6565	105914	13389
27	2022	8635	107324	-5780
28	2023	8328	115387	1928

	Year	Enterprises_Ireland_250+	Enterprises_Dublin_250+	Enterprises_Ireland	Enterprises_Dublin
0	2008	525	318	244195	70743
1	2009	489	298	244428	73833
2	2010	463	287	242692	74392
3	2011	462	277	240880	74421
4	2012	478	288	244394	76547

	COUNTRY	Country	BIRTH	Place of birth	GENDER	Gender	RATE	Rate	YEAR	Year	Unit Code	Unit	PowerCode	Reference Period Code	Reference Period	Value	Flag Codes	Flags
0	AUS	Australia	FB	Foreign-born	MEN	Men	N_RATE	Employment rate	2000	2000	PC	Percentage	Units	NaN	NaN	74.0	NaN	NaN
1	AUS	Australia	FB	Foreign-born	MEN	Men	N_RATE	Employment rate	2001	2001	PC	Percentage	Units	NaN	NaN	73.9	NaN	NaN
2	AUS	Australia	FB	Foreign-born	MEN	Men	N_RATE	Employment rate	2002	2002	PC	Percentage	Units	NaN	NaN	74.0	NaN	NaN
3	AUS	Australia	FB	Foreign-born	MEN	Men	N_RATE	Employment rate	2003	2003	PC	Percentage	Units	NaN	NaN	74.1	NaN	NaN
4	AUS	Australia	FB	Foreign-born	MEN	Men	N_RATE	Employment rate	2004	2004	PC	Percentage	Units	NaN	NaN	75.6	NaN	NaN

	Statistic Label	Year	Item	UNIT	VALUE
0	General Government Transactions ESA 2010: Reve...	1995	General Government transactions - Revenue - ES...	Euro Million	20959
1	General Government Transactions ESA 2010: Reve...	1995	General Government transactions - Revenue - Ta...	Euro Million	18262
2	General Government Transactions ESA 2010: Reve...	1995	General Government transactions - Revenue - Ta...	Euro Million	14933
3	General Government Transactions ESA 2010: Reve...	1995	General Government transactions - Revenue - So...	Euro Million	3329
4	General Government Transactions ESA 2010: Reve...	1995	General Government transactions - Revenue - Sa...	Euro Million	982

	Statistic Label	Year	Employment Size	County	UNIT	VALUE
99	Active Enterprises	2020	All persons engaged size classes	Limerick	Number	9497
100	Active Enterprises	2020	250 and over	All Counties	Number	666
101	Active Enterprises	2020	250 and over	Dublin	Number	413
102	Active Enterprises	2020	250 and over	Cork	Number	56
103	Active Enterprises	2020	250 and over	Limerick	Number	14

	COUB	Country of birth	FBORN	Place of birth	EDU	Education level	SECT	Sector	SEX	Sex	COU	Country of residence	Value	Flag Codes	Flags
0	AFRI	Africa	1	Foreign-born	2	ISCED 3/4	E	Electricity, gas and water supply	2	Women	IRL	Ireland	3	NaN	NaN
1	AFRI	Africa	1	Foreign-born	2	ISCED 3/4	E	Electricity, gas and water supply	1	Men	IRL	Ireland	3	NaN	NaN
2	AFRI	Africa	1	Foreign-born	2	ISCED 3/4	E	Electricity, gas and water supply	ALL_SEX	Men and women	IRL	Ireland	6	NaN	NaN
3	AFRI	Africa	1	Foreign-born	2	ISCED 3/4	G	Wholesale and retail trade; repair of motor ve...	2	Women	IRL	Ireland	131	NaN	NaN
4	AFRI	Africa	1	Foreign-born	2	ISCED 3/4	G	Wholesale and retail trade; repair of motor ve...	1	Men	IRL	Ireland	217	NaN	NaN

	INDICATOR	Indicator	FAM_TYPE	Household type	COU	Country	YEA	Year	Unit Code	Unit	PowerCode	Reference Period Code	Reference Period	Value	Flag Codes	Flags
0	5_1	Total gross earnings before taxes in US dollar...	MARRIED2	Two-earner married couple, one at 100% of aver...	AUS	Australia	2000	2000	USD	US Dollar	Units	NaN	NaN	52616.675635	NaN	NaN
1	5_1	Total gross earnings before taxes in US dollar...	MARRIED2	Two-earner married couple, one at 100% of aver...	AUS	Australia	2001	2001	USD	US Dollar	Units	NaN	NaN	54084.089270	NaN	NaN
2	5_1	Total gross earnings before taxes in US dollar...	MARRIED2	Two-earner married couple, one at 100% of aver...	AUS	Australia	2002	2002	USD	US Dollar	Units	NaN	NaN	56595.254697	NaN	NaN
3	5_1	Total gross earnings before taxes in US dollar...	MARRIED2	Two-earner married couple, one at 100% of aver...	AUS	Australia	2003	2003	USD	US Dollar	Units	NaN	NaN	59102.782366	NaN	NaN
4	5_1	Total gross earnings before taxes in US dollar...	MARRIED2	Two-earner married couple, one at 100% of aver...	AUS	Australia	2004	2004	USD	US Dollar	Units	NaN	NaN	61459.639570	NaN	NaN

	Year	Country	Household type	Value
31165	2000	Ireland	Two-earner married couple, one at 100% of aver...	19.744692
31166	2001	Ireland	Two-earner married couple, one at 100% of aver...	15.675248
31167	2002	Ireland	Two-earner married couple, one at 100% of aver...	14.082765
31168	2003	Ireland	Two-earner married couple, one at 100% of aver...	14.890699
31169	2004	Ireland	Two-earner married couple, one at 100% of aver...	15.583686

Country	Year	EU in OECD		Ireland		United Kingdom		United States
Household type		Tax Married 100% w 2 Children	Tax Single 100% wo Child	Tax Married 100% w 2 Children	Tax Single 100% wo Child	Tax Married 100% w 2 Children	Tax Single 100% wo Child	Tax Married 100% w 2 Children	Tax Single 100% wo Child
0	2000	14.690797	17.986737	19.744692	24.076818	16.485056	17.395022	12.836110	17.260230
1	2001	14.218244	17.673126	15.675248	20.348832	15.139431	17.275557	12.454022	17.175813
2	2002	13.932074	17.402316	14.082765	18.708792	15.262469	17.359484	12.365608	17.008386
3	2003	13.638616	17.248319	14.890699	19.693734	15.489576	17.536957	10.487830	16.767553
4	2004	13.737779	17.434849	15.583686	20.272378	15.626080	17.611695	10.515535	16.722431

	Month	Year	Ireland 10Y Bond Yield	Percentage Change MoM	Change MoM
0	June	1999	4,743	NaN	NaN
1	July	1999	5,082	7,147375079	0,339
2	August	1999	5,202	2,361275089	0,12
3	September	1999	5,338	2,614379085	0,136
4	October	1999	5,394	1,049082053	0,056

Index	Year	Ireland 10Y Bond Yield
0	1999	5.251000
1	2000	5.478000
2	2001	5.012750
3	2002	4.981250
4	2003	4.118083

	Year	Inflation Consumer Prices (Annual %)	GDP Growth (annual %)	GDP per Capita Growth (Annual %)	Inflation GDP Deflator (Annual %)
0	1960	0.5	NaN	NaN	NaN
1	1961	2.8	NaN	NaN	NaN
2	1962	4.3	NaN	NaN	NaN
3	1963	2.5	NaN	NaN	NaN
4	1964	6.7	NaN	NaN	NaN

	Series Name	Series Code	Country Name	Country Code	1960 [YR1960]	1961 [YR1961]	1962 [YR1962]	1963 [YR1963]	1964 [YR1964]	1965 [YR1965]	...	2014 [YR2014]	2015 [YR2015]	2016 [YR2016]	2017 [YR2017]	2018 [YR2018]	2019 [YR2019]	2020 [YR2020]	2021 [YR2021]	2022 [YR2022]	2023 [YR2023]
Population	total	SP.POP.TOTL	Afghanistan	AFG	8622466	8790140	8969047	9157465	9355514	9565147	...	32716210	33753499	34636207	35643418	36686784	37769499	38972230	40099462	41128771	..
Population	total	SP.POP.TOTL	Albania	ALB	1608800	1659800	1711319	1762621	1814135	1864791	...	2889104	2880703	2876101	2873457	2866376	2854191	2837849	2811666	2777689	..
Population	total	SP.POP.TOTL	Algeria	DZA	11394307	11598608	11778260	11969451	12179099	12381256	...	38760168	39543154	40339329	41136546	41927007	42705368	43451666	44177969	44903225	..
Population	total	SP.POP.TOTL	American Samoa	ASM	20085	20626	21272	21949	22656	23391	...	52217	51368	50448	49463	48424	47321	46189	45035	44273	..
Population	total	SP.POP.TOTL	Andorra	AND	9443	10216	11014	11839	12690	13563	...	71621	71746	72540	73837	75013	76343	77700	79034	79824	..

Load Libraries¶

Possible Data Table (combined datasets)¶

CSO Datasets¶

PEA 15 (Births, Deaths, Migration by Year)¶

Import and Clean¶

Graphs¶

PEA 03 (Age, Sex, Migration by Year)¶

Import and Clean¶

Graphs¶

PEA 18 (Country and Mirgation by Year)¶

Import and Clean¶

Graphs¶

Further filtering¶

!!! PEA 23 (Emigration, Sex, Country by Year)¶

Import and Clean¶

!!! PEA 24 (Immigration, Sex, Citizenship)¶

Import and Clean¶

CPM01 (Consumer Price Index)¶

Import and Clean¶

Graphs¶

TSA 02 (Import, Export)¶

Import and Clean¶

Graphs¶

BPA 33 (Direct Investment)¶

Import and Clean¶

Graphs¶

Further Filtering¶

GFA 01 (General Government Transactions)¶

Import and Clean¶

Graphs¶

BRA 08 (Number of Enterprises)¶

Import and Clean¶

Graphs¶

Filtering & Cleaning 2.0¶

OECD Datasets¶

OECD 1 (Country, Sex, Employment)¶

Import and Clean¶

Graphs¶

Native Irish¶

Irish and Other Countries¶

Foreign Irish¶

Native Canadians¶

OECD 2 (Immigrants, Sector)¶

Import and Clean¶

Graphs¶

OECD 3 (Income Tax Married100 w 2 Children)¶

Import and Clean¶

Graphs¶

Further Filtering¶

Trading Economics Datasets¶

TE01 (10Y Irish Bond Yields)¶

Import and Clean¶

Graphs¶

Data World Bank Datasets¶

WB01 (GDP Growth Annual %)¶

Import and Clean¶

Graphs¶

WB02 (GDP per Capita Growth Annual %)¶

Import and Clean¶

Graphs¶

WB03 (Inflation Consumer Prices Annual %)¶

Import and Clean¶

Graphs¶

WB04 (Inflation GDP Deflator Annual %)¶

Import and Clean¶

Graphs¶

Combined WB Datasets¶

WB05 (Population by Country)¶

Import and Clean¶

Graphs¶

More Cleaning¶

Combining WB with PEA15¶

Combine WB_PEA15 with OECD1 variables¶

Combine Natives Outside Ireland¶

Combine PEA03 with main dataframe¶

Combine 15-24 Ages Migration to Main Dataframe¶

Combine 25-44 Ages Migration to Main Dataframe¶

Combine CPM01 with Main Dataframe¶

Housing CPI¶

Food & NA_Drinks CPI¶

	Country	Year	Population
16981	East Asia & Pacific	2023	..
16988	European Union	2023	..
17009	North America	2023	..
17011	OECD members	2023	..
17017	South Asia	2023	..

Country	Year	East Asia & Pacific	European Union	Ireland	South Asia	United Kingdom
58	2018	2.341384e+09	447001100.0	4867316.0	1.840534e+09	66460344.0
59	2019	2.353857e+09	447367191.0	4934340.0	1.861599e+09	66836327.0
60	2020	2.363934e+09	447692315.0	4985382.0	1.882532e+09	67081234.0
61	2021	2.370190e+09	447178112.0	5033164.0	1.901912e+09	67026292.0
62	2022	2.375162e+09	447370510.0	5127170.0	1.919348e+09	66971395.0

	Year	Participation rate_Ireland_Total_Foreign-born	Participation rate_Ireland_Total_Native-born	Participation rate_Ireland_Women_Foreign-born	Participation rate_Ireland_Women_Native-born	Participation rate_United Kingdom_Total_Foreign-born	Participation rate_United Kingdom_Total_Native-born	Participation rate_United Kingdom_Women_Foreign-born	Participation rate_United Kingdom_Women_Native-born	Unemployment rate_Ireland_Total_Foreign-born	Unemployment rate_Ireland_Total_Native-born	Unemployment rate_Ireland_Women_Foreign-born	Unemployment rate_Ireland_Women_Native-born	Unemployment rate_United Kingdom_Total_Foreign-born	Unemployment rate_United Kingdom_Total_Native-born	Unemployment rate_United Kingdom_Women_Foreign-born	Unemployment rate_United Kingdom_Women_Native-born
13	2013	71.9	69.3	63.3	62.5	74.1	76.9	65.4	72.0	15.7	12.7	14.5	9.8	8.9	7.5	9.8	6.7
14	2014	70.8	69.5	62.2	62.7	74.7	77.1	66.1	72.4	13.5	11.0	12.7	8.6	7.1	6.1	8.2	5.5
15	2015	70.7	69.8	61.8	63.0	75.4	77.3	67.4	72.7	11.4	9.1	9.7	7.2	6.4	5.2	7.3	4.8
16	2016	71.8	70.2	62.9	64.0	76.1	77.6	67.6	73.3	9.3	7.7	8.8	6.0	5.5	4.8	6.6	4.4
17	2017	75.2	71.9	67.6	66.3	76.4	77.9	68.1	74.0	8.2	6.5	8.5	5.8	5.1	4.3	6.5	3.8
18	2018	76.2	71.9	68.9	66.6	77.3	78.0	69.4	74.2	7.2	5.4	8.2	5.0	4.7	4.0	5.6	3.7
19	2019	75.9	72.4	68.6	67.0	78.0	78.2	70.5	74.6	5.9	4.8	6.2	4.3	4.3	3.7	4.9	3.3
20	2020	72.8	70.1	65.0	64.8	NaN	NaN	NaN	NaN	7.5	5.3	8.0	5.2	NaN	NaN	NaN	NaN
21	2021	77.6	73.6	71.5	69.3	79.9	78.7	74.6	75.5	7.5	5.9	7.6	5.7	5.6	3.3	4.5	2.9
22	2022	81.0	75.2	75.0	70.6	79.7	78.3	74.3	75.1	4.9	4.4	6.3	4.1	4.9	3.5	5.5	3.2

	Year	Annual births	Annual deaths	Emigrants	Immigrants	Net migration	Inflation Consumer Prices (Annual %)	GDP Growth (annual %)	GDP per Capita Growth (Annual %)	Inflation GDP Deflator (Annual %)	...	Participation rate_United Kingdom_Women_Foreign-born	Participation rate_United Kingdom_Women_Native-born	Unemployment rate_Ireland_Total_Foreign-born	Unemployment rate_Ireland_Total_Native-born	Unemployment rate_Ireland_Women_Foreign-born	Unemployment rate_Ireland_Women_Native-born	Unemployment rate_United Kingdom_Total_Foreign-born	Unemployment rate_United Kingdom_Total_Native-born	Unemployment rate_United Kingdom_Women_Foreign-born	Unemployment rate_United Kingdom_Women_Native-born
68	2019	60500.0	30900.0	53100.0	97100.0	44000.0	0.9	5.3	3.9	3.4	...	70.5	74.6	5.9	4.8	6.2	4.3	4.3	3.7	4.9	3.3
69	2020	59100.0	32400.0	50900.0	95600.0	44700.0	-0.3	6.6	5.5	-1.2	...	NaN	NaN	7.5	5.3	8.0	5.2	NaN	NaN	NaN	NaN
70	2021	57300.0	34300.0	52300.0	74100.0	21800.0	2.4	15.1	14.0	0.5	...	74.6	75.5	7.5	5.9	7.6	5.7	5.6	3.3	4.5	2.9
71	2022	59700.0	33600.0	56100.0	107800.0	51700.0	7.8	9.4	7.4	6.6	...	74.3	75.1	4.9	4.4	6.3	4.1	4.9	3.5	5.5	3.2
72	2023	55500.0	35500.0	64000.0	141600.0	77600.0	NaN	NaN	NaN	NaN	...	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN

	Year	Annual births	Annual deaths	Emigrants	Immigrants	Net migration	Inflation Consumer Prices (Annual %)	GDP Growth (annual %)	GDP per Capita Growth (Annual %)	Inflation GDP Deflator (Annual %)	...	Participation rate_United Kingdom_Women_Foreign-born	Participation rate_United Kingdom_Women_Native-born	Unemployment rate_Ireland_Total_Foreign-born	Unemployment rate_Ireland_Total_Native-born	Unemployment rate_Ireland_Women_Foreign-born	Unemployment rate_Ireland_Women_Native-born	Unemployment rate_United Kingdom_Total_Foreign-born	Unemployment rate_United Kingdom_Total_Native-born	Unemployment rate_United Kingdom_Women_Foreign-born	Unemployment rate_United Kingdom_Women_Native-born
63	2014	68400.0	29200.0	75000.0	66500.0	-8500.0	0.2	8.8	8.0	0.2	...	66.1	72.4	13.5	11.0	12.7	8.6	7.1	6.1	8.2	5.5
64	2015	66400.0	29900.0	70000.0	75900.0	5900.0	-0.3	24.5	23.3	8.3	...	67.4	72.7	11.4	9.1	9.7	7.2	6.4	5.2	7.3	4.8
65	2016	65400.0	29800.0	66200.0	82300.0	16200.0	0.0	1.8	0.6	0.6	...	67.6	73.3	9.3	7.7	8.8	6.0	5.5	4.8	6.6	4.4
66	2017	63400.0	31300.0	56100.0	95300.0	39200.0	0.3	9.3	8.1	1.3	...	68.1	74.0	8.2	6.5	8.5	5.8	5.1	4.3	6.5	3.8
67	2018	61600.0	32000.0	51600.0	96000.0	44400.0	0.5	8.5	7.1	1.1	...	69.4	74.2	7.2	5.4	8.2	5.0	4.7	4.0	5.6	3.7
68	2019	60500.0	30900.0	53100.0	97100.0	44000.0	0.9	5.3	3.9	3.4	...	70.5	74.6	5.9	4.8	6.2	4.3	4.3	3.7	4.9	3.3
69	2020	59100.0	32400.0	50900.0	95600.0	44700.0	-0.3	6.6	5.5	-1.2	...	NaN	NaN	7.5	5.3	8.0	5.2	NaN	NaN	NaN	NaN
70	2021	57300.0	34300.0	52300.0	74100.0	21800.0	2.4	15.1	14.0	0.5	...	74.6	75.5	7.5	5.9	7.6	5.7	5.6	3.3	4.5	2.9
71	2022	59700.0	33600.0	56100.0	107800.0	51700.0	7.8	9.4	7.4	6.6	...	74.3	75.1	4.9	4.4	6.3	4.1	4.9	3.5	5.5	3.2
72	2023	55500.0	35500.0	64000.0	141600.0	77600.0	NaN	NaN	NaN	NaN	...	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN

Index	Year	Avg Employment Natives outside Ireland	Avg Participation Natives outside Ireland	Avg Unemployment Natives outside Ireland
0	2000	67.085714	71.690476	6.623810
1	2001	66.992308	71.676923	6.730769
2	2002	67.226923	72.092308	6.942308
3	2003	67.007692	72.080769	7.215385
4	2004	67.061538	72.246154	7.346154

Index	Year	Avg Employment Native Women outside Ireland	Avg Participation Native Women outside Ireland	Avg Unemployment Native Women outside Ireland
0	2000	59.819048	64.476190	7.642857
1	2001	60.046154	65.086957	7.653846
2	2002	60.411538	65.537500	7.707692
3	2003	60.484615	65.756000	7.884615
4	2004	60.784615	65.834615	7.961538

Index	Year	Emigrants 15-24 years	Immigrants 15-24 years	Net migration 15-24 years
0	1987	16000.000000	3400.000000	NaN
1	1988	20833.333333	3600.000000	NaN
2	1989	24666.666667	5133.333333	NaN
3	1990	20533.333333	6733.333333	NaN
4	1991	13266.666667	6200.000000	NaN

Index	Year	Emigrants 25-44 years	Immigrants 25-44 years	Net migration 25-44 years
0	1987	7866.666667	4066.666667	NaN
1	1988	12200.000000	4800.000000	NaN
2	1989	14600.000000	7033.333333	NaN
3	1990	11266.666667	9300.000000	NaN
4	1991	7000.000000	9700.000000	NaN

Index	Year	Housing_CPI
29	1996	NaN
42	1997	1.200000
55	1998	0.733333
68	1999	-6.583333
81	2000	9.608333